packcr 0.0.7 → 0.1.0

data/lib/packcr/templates/context/source.rs.erb

@@ -0,0 +1,625 @@
+/* A packrat parser generated by PackCR <%= Packcr::VERSION %> */
+
+use std::cell::RefCell;
+use std::collections::{HashMap, HashSet};
+use std::io::Read;
+use std::rc::Rc;
+
+<%- if !code(:esource).empty? -%>
+
+<%-   code(:esource).each do |code| -%>
+<%=     stream.get_code_block(code, 0, @iname) -%>
+<%-   end -%>
+<%- end -%>
+<%- use_value = @root.rules.map {|r| r.vars.size }.max > 0 -%>
+
+struct LrMemoTable {
+    memos: HashMap<usize, LrMemoMap>,
+}
+impl LrMemoTable {
+    fn new() -> Self {
+        Self {
+            memos: HashMap::new(),
+        }
+    }
+
+    fn clear(&mut self) {
+        self.memos.clear();
+    }
+
+    fn set(&mut self, index: usize, rule: Rule, memo: LrMemo) {
+        let memo_map = self.memos.entry(index).or_default();
+        memo_map.insert(rule, memo);
+    }
+
+    fn get(&mut self, index: usize, rule: Rule) -> &mut LrMemo {
+        self.memos.get_mut(&index).unwrap().get_mut(&rule).unwrap()
+    }
+    fn has(&mut self, index: usize, rule: Rule) -> bool {
+        let memo_opt = self.memos.get_mut(&index);
+        if memo_opt.is_none() {
+            return false;
+        }
+        memo_opt.unwrap().contains_key(&rule)
+    }
+}
+
+struct LrMemo {
+    offset: usize,
+    answer: Option<Rc<RefCell<ThunkChunk>>>,
+    grow: bool,
+}
+
+impl LrMemo {
+    fn new(offset: usize) -> Self {
+        Self {
+            offset,
+            answer: None,
+            grow: false,
+        }
+    }
+
+    fn start_grow(&mut self) -> bool {
+        if self.grow || self.answer.is_some() {
+            return false;
+        }
+        self.answer = None;
+        self.grow = true;
+        true
+    }
+
+    fn start_match(&mut self, answer: Option<ThunkChunk>, offset: usize) {
+        self.answer = answer.map(|c| Rc::new(RefCell::new(c)));
+        self.offset = offset;
+    }
+
+    fn answer_as_deref_mut(&mut self) -> Option<Rc<RefCell<ThunkChunk>>> {
+        self.answer.as_ref()?;
+        self.answer.as_ref().map(Rc::clone)
+    }
+}
+
+struct ThunkChunk {
+    thunks: Rc<RefCell<Vec<Thunk>>>,
+    capts: CaptureTable,
+    pos: usize,
+    values: ValueTable,
+}
+
+impl ThunkChunk {
+    fn new(pos: usize) -> Self {
+        Self {
+            values: ValueTable::new(),
+            capts: CaptureTable::new(),
+            thunks: Rc::new(RefCell::new(Vec::new())),
+            pos,
+        }
+    }
+
+    fn push_leaf(
+        &self,
+        action: Action,
+        end: usize,
+        value_indices: &[usize],
+        capt_indices: &[usize],
+    ) {
+        {
+            let start = self.pos;
+            let mut value_refs = HashMap::new();
+            for &index in value_indices {
+                value_refs.insert(index, ValueRef::new(index, self.values.clone()));
+            }
+            let mut capts = HashMap::new();
+            for &index in capt_indices {
+                capts.insert(index, self.capts[index].clone());
+            }
+            let leaf = Thunk::Leaf(ThunkLeaf::new(action, start, end, value_refs, capts));
+            self.thunks.borrow_mut().push(leaf);
+        }
+    }
+
+    fn value_ref(&self, index: usize) -> ValueRef {
+        ValueRef::new(index, self.values.clone())
+    }
+}
+
+enum Thunk {
+    Leaf(ThunkLeaf),
+    Node(ThunkNode),
+}
+impl Thunk {
+    fn do_action(&self, processor: &ThunkProcessor, value: ValueRef) {
+        match self {
+            Thunk::Leaf(leaf) => leaf.do_action(processor, value),
+            Thunk::Node(node) => node.do_action(processor),
+        }
+    }
+}
+
+#[allow(dead_code)]
+struct ThunkLeaf {
+    value_refs: HashMap<usize, ValueRef>,
+    capts: HashMap<usize, Capture>,
+    capt0: Capture,
+    action: Action,
+}
+impl ThunkLeaf {
+    fn new(
+        action: Action,
+        start: usize,
+        end: usize,
+        value_refs: HashMap<usize, ValueRef>,
+        capts: HashMap<usize, Capture>,
+    ) -> Self {
+        Self {
+            value_refs,
+            capts,
+            capt0: Capture { start, end },
+            action,
+        }
+    }
+
+    fn do_action(&self, processor: &ThunkProcessor, mut value: ValueRef) {
+        value.with_mut(|v| {
+            processor.call_action(self.action, self, v);
+        });
+    }
+    <%- if use_value -%>
+
+    fn values(&self) -> HashMap<usize, Value> {
+        self.value_refs.iter().map(|(k, v)| (*k, v.get())).collect()
+    }
+    <%- end -%>
+}
+
+struct ThunkNode {
+    thunks: Rc<RefCell<Vec<Thunk>>>,
+    value: ValueRef,
+}
+impl ThunkNode {
+    fn do_action(&self, processor: &ThunkProcessor) {
+        for thunk in self.thunks.borrow().iter() {
+            thunk.do_action(processor, self.value.clone());
+        }
+    }
+}
+
+#[derive(Clone)]
+struct Capture {
+    start: usize,
+    end: usize,
+}
+
+type Value = i32;
+
+struct ValueTable {
+    table: Rc<RefCell<HashMap<usize, Value>>>,
+}
+impl ValueTable {
+    fn new() -> Self {
+        Self {
+            table: Rc::new(RefCell::new(HashMap::new())),
+        }
+    }
+    <%- if use_value -%>
+
+    fn clear(&mut self) {
+        self.table.borrow_mut().clear();
+    }
+    <%- end -%>
+
+    fn with_mut<F>(&mut self, index: usize, f: F)
+    where
+        F: FnOnce(&mut Value),
+    {
+        let mut table = self.table.borrow_mut();
+        let value = table.entry(index).or_insert(0);
+        f(value);
+    }
+}
+impl Clone for ValueTable {
+    fn clone(&self) -> Self {
+        Self {
+            table: self.table.clone(),
+        }
+    }
+}
+
+struct ValueRef {
+    values: ValueTable,
+    index: usize,
+}
+impl ValueRef {
+    fn new(index: usize, values: ValueTable) -> Self {
+        Self { index, values }
+    }
+
+    fn with_mut<F>(&mut self, f: F)
+    where
+        F: FnOnce(&mut i32),
+    {
+        self.values.with_mut(self.index, f);
+    }
+
+    <%- if use_value -%>
+    fn get(&self) -> i32 {
+        *self.values.table.borrow().get(&self.index).unwrap_or(&0)
+    }
+    <%- end -%>
+}
+impl Clone for ValueRef {
+    fn clone(&self) -> Self {
+        Self {
+            index: self.index,
+            values: self.values.clone(),
+        }
+    }
+}
+
+struct CaptureTable {
+    table: HashMap<usize, Capture>,
+    len: usize,
+}
+
+type RuleSet = HashSet<Rule>;
+
+type LrMemoMap = HashMap<Rule, LrMemo>;
+
+impl CaptureTable {
+    fn new() -> Self {
+        Self {
+            table: HashMap::new(),
+            len: 0,
+        }
+    }
+
+    fn resize(&mut self, len: usize) {
+        let current_len = self.len;
+        if len > current_len {
+            for i in current_len..len {
+                self.table.insert(i, Capture { start: 0, end: 0 });
+            }
+        } else if len < current_len {
+            for i in len..current_len {
+                self.table.remove(&i);
+            }
+        }
+        self.len = len;
+    }
+}
+impl std::ops::Index<usize> for CaptureTable {
+    type Output = Capture;
+
+    fn index(&self, index: usize) -> &Self::Output {
+        &self.table[&index]
+    }
+}
+impl std::ops::IndexMut<usize> for CaptureTable {
+    fn index_mut(&mut self, index: usize) -> &mut Self::Output {
+        if self.len <= index {
+            self.resize(index + 1);
+        }
+        self.table.get_mut(&index).unwrap()
+    }
+}
+
+struct Input {
+    input: Box<dyn std::io::Read>,
+    start_position: usize,
+    position_offset: usize,
+    buffer: String,
+}
+
+impl Input {
+    fn new(input: impl std::io::Read + 'static) -> Self {
+        Self {
+            input: Box::new(input),
+            start_position: 0,
+            position_offset: 0,
+            buffer: String::new(),
+        }
+    }
+
+    fn refill_buffer(&mut self, num: usize) -> usize {
+        let mut len = self.buffer.len();
+        if len >= self.position_offset + num {
+            return len - self.position_offset;
+        }
+
+        let mut input_buffer = [0u8; 1024];
+
+        while len < self.position_offset + num {
+            match self.input.read(&mut input_buffer) {
+                Ok(0) => break,
+                Ok(bytes_read) => {
+                    let s = std::string::String::from_utf8_lossy(&input_buffer[..bytes_read]);
+                    self.buffer.push_str(&s);
+                }
+                Err(_) => break,
+            }
+            len = self.buffer.len();
+        }
+
+        len - self.position_offset
+    }
+
+    fn commit_buffer(&mut self) {
+        let position_offset = self.position_offset;
+
+        self.buffer.drain(..position_offset);
+
+        self.start_position += position_offset;
+        self.position_offset = 0;
+    }
+
+    #[allow(dead_code)]
+    fn get_char_as_utf32(&mut self) -> (i32, usize) {
+        if self.refill_buffer(1) < 1 {
+            return (0, 0);
+        }
+
+        let current_position = self.position_offset;
+        let remaining_chars: &str = &self.buffer[current_position..];
+
+        if let Some(ch) = remaining_chars.chars().next() {
+            let bytes_used = ch.len_utf8();
+
+            if self.refill_buffer(bytes_used) < bytes_used {
+                return (0, 0);
+            }
+
+            let utf32_value = ch as u32 as i32;
+
+            return (utf32_value, bytes_used);
+        }
+
+        (0, 0)
+    }
+
+    fn back_to(&mut self, memo: &mut LrMemo) -> Option<Rc<RefCell<ThunkChunk>>> {
+        self.position_offset = memo.offset;
+        memo.answer_as_deref_mut()
+    }
+}
+
+struct <%= class_name %> {
+    level: usize,
+    memos: LrMemoTable,
+    input: Input,
+}
+
+impl <%= class_name %> {
+    fn new(input: impl std::io::Read + 'static) -> Self {
+        Self {
+            level: 0,
+            memos: LrMemoTable::new(),
+            input: Input::new(input),
+        }
+    }
+
+    fn error(&self) -> ! {
+        eprintln!("Syntax error");
+        std::process::exit(1);
+    }
+
+    fn refill_buffer(&mut self, num: usize) -> usize {
+        self.input.refill_buffer(num)
+    }
+
+    fn commit_buffer(&mut self) {
+        self.input.commit_buffer();
+        self.memos.clear();
+    }
+
+    fn parse(&mut self) -> bool {
+        let mut answer = ThunkChunk::new(0);
+        let pos = self.input.start_position;
+
+        <%- if !@root.rules.empty? -%>
+        let apply_result = self.apply_rule(
+            Rule::<%= Packcr.camelize(@root.rules[0].name) %>,
+            &mut answer,
+            0,
+            self.input.position_offset,
+            None,
+        );
+
+        if apply_result {
+            if let Thunk::Node(node) = answer.thunks.borrow_mut().last().unwrap() {
+                node.do_action(&ThunkProcessor::new(&self.input.buffer));
+            }
+        } else {
+            self.error();
+        }
+
+        <%- end -%>
+        self.commit_buffer();
+        pos != self.input.start_position && self.refill_buffer(1) >= 1
+    }
+
+    #[allow(dead_code)]
+    fn get_char_as_utf32(&mut self) -> (i32, usize) {
+        self.input.get_char_as_utf32()
+    }
+
+    fn grow_lr(&mut self, rule: Rule, offset: usize) {
+        loop {
+            let old_offset = self.input.position_offset;
+            self.input.position_offset = offset;
+
+            let answer = self.call_rule(rule, offset, Some(RuleSet::new()));
+            if answer.is_none() || self.input.position_offset <= old_offset {
+                return;
+            }
+
+            let memo = self.memos.get(offset, rule);
+            memo.start_match(answer, self.input.position_offset);
+        }
+    }
+
+    fn rule_answer(&mut self, rule: Rule) -> Option<Rc<RefCell<ThunkChunk>>> {
+        let offset = self.input.position_offset;
+        if !self.memos.has(offset, rule) {
+            let memo = LrMemo::new(offset);
+            self.memos.set(offset, rule, memo);
+            let answer = self.call_rule(rule, offset, None);
+
+            let memo = self.memos.get(offset, rule);
+            memo.start_match(answer, self.input.position_offset);
+            if !memo.grow {
+                return self.input.back_to(memo);
+            }
+            self.grow_lr(rule, offset);
+
+            let memo = self.memos.get(offset, rule);
+            memo.grow = false;
+            return self.input.back_to(memo);
+        }
+
+        let memo = self.memos.get(offset, rule);
+        if memo.start_grow() {
+            return None;
+        }
+        self.input.back_to(memo)
+    }
+
+    fn rule_answer_with_limits(
+        &mut self,
+        rule: Rule,
+        limits: RuleSet,
+    ) -> Option<Rc<RefCell<ThunkChunk>>> {
+        let offset = self.input.position_offset;
+        let answer = self.call_rule(rule, offset, Some(limits));
+        if !self.memos.has(offset, rule) {
+            answer.as_ref()?;
+            return Some(Rc::new(RefCell::new(answer.unwrap())));
+        }
+        let memo = self.memos.get(offset, rule);
+
+        if self.input.position_offset > memo.offset {
+            memo.start_match(answer, self.input.position_offset);
+            return memo.answer_as_deref_mut();
+        }
+
+        self.input.back_to(memo)
+    }
+
+    fn apply_rule(
+        &mut self,
+        rule: Rule,
+        parent: &mut ThunkChunk,
+        value_index: usize,
+        offset: usize,
+        limits: Option<RuleSet>,
+    ) -> bool {
+        let answer = match &limits {
+            Some(limit_set)
+                if self.input.position_offset == offset && !limit_set.contains(&rule) =>
+            {
+                self.rule_answer_with_limits(rule, limits.unwrap())
+            }
+            _ => self.rule_answer(rule),
+        };
+
+        let Some(answer) = answer else {
+            return false;
+        };
+
+        let new_node = Thunk::Node(ThunkNode {
+            thunks: answer.borrow().thunks.clone(),
+            value: parent.value_ref(value_index),
+        });
+        parent.thunks.borrow_mut().push(new_node);
+
+        true
+    }
+
+    fn call_rule(
+        &mut self,
+        rule: Rule,
+        offset: usize,
+        mut limits: Option<RuleSet>,
+    ) -> Option<ThunkChunk> {
+        if let Some(ref mut limits) = limits {
+            limits.insert(rule);
+        }
+        match rule {
+            <%- @root.rules.each do |rule| -%>
+            Rule::<%= Packcr.camelize(rule.name) %> => self.evaluate_rule_<%= rule.name %>(offset, limits),
+            <%- end -%>
+        }
+    }
+
+    <%- @root.rules.each do |rule| -%>
+
+    <%-   gen = ::Packcr::Generator.new(rule, @ascii, @location, :rs) -%>
+    <%- %><%= gen.generate_code(rule, 0, 4, false) -%>
+    <%- end -%>
+}
+
+struct ThunkProcessor<'a> {
+    buffer: &'a str,
+}
+
+impl<'a> ThunkProcessor<'a> {
+    fn new(buffer: &'a str) -> Self {
+        Self { buffer }
+    }
+
+    fn call_action(&self, action: Action, thunk: &ThunkLeaf, value: &mut i32) {
+        match action {
+            <%- @root.rules.each do |rule| -%>
+            <%-   rule.codes.each do |code| -%>
+            Action::<%= Packcr.camelize(rule.name) %><%= code.index %> => self.action_<%= rule.name %>_<%= code.index %>(thunk, value),
+            <%-   end -%>
+            <%- end -%>
+        }
+    }
+    <%- @root.rules.each do |rule| -%>
+    <%-   rule.codes.each do |code| -%>
+
+    #[allow(unused_variables,non_snake_case)]
+    fn action_<%= rule.name %>_<%= code.index %>(&self, leaf: &ThunkLeaf, out: &mut i32) {
+        <%-   code.vars.each_with_index do |ref, i| -%>
+        <%-     if i == 0 -%>
+        let values = leaf.values();
+        <%-     end -%>
+        let <%= ref.var %> = values[&<%= ref.index %>];
+        <%-   end -%>
+        let _0 = {
+            let capt = &leaf.capt0;
+            &self.buffer[(capt.start)..(capt.end)]
+        };
+        <%-   code.capts.size.times do |i| -%>
+        let _<%= i + 1 %> = {
+            let capt = &leaf.capts[&<%= i %>];
+            &self.buffer[(capt.start)..(capt.end)]
+        };
+        <%-   end -%>
+    <%=   stream.get_code_block(code.code, 4, @iname) -%>
+    }
+    <%-   end -%>
+    <%- end -%>
+}
+
+#[derive(Copy, Clone)]
+enum Action {
+    <%- @root.rules.each do |rule| -%>
+    <%-   rule.codes.each do |code| -%>
+    <%= Packcr.camelize(rule.name) %><%= code.index %>,
+    <%-   end -%>
+    <%- end -%>
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash)]
+enum Rule {
+  <%- @root.rules.each do |rule| -%>
+    <%= Packcr.camelize(rule.name) %>,
+  <%- end -%>
+}
+<%- if !code(:lsource).empty? -%>
+
+<%-   code(:lsource).each do |code| -%>
+<%=     stream.get_code_block(code, 0, @iname) -%>
+<%-   end -%>
+<%- end -%>

data/lib/packcr/templates/node/action.c.erb

@@ -1,5 +1,5 @@
 {
-    pcc_thunk_t *const thunk = pcc_thunk__create_leaf(ctx->auxil, pcc_action_<%= gen.rule.name %>_<%= index %>, <%= gen.rule.vars.length %>, <%= gen.rule.capts.length %>);
+    packcr_thunk_t *const thunk = packcr_thunk__create_leaf(ctx->auxil, packcr_action_<%= gen.rule.name %>_<%= index %>, <%= gen.rule.vars.length %>, <%= gen.rule.capts.length %>);
 <%- vars.each do |var| -%>
     thunk->data.leaf.values.buf[<%= var.index %>] = &(chunk->values.buf[<%= var.index %>]);
 <%- end -%>
@@ -12,5 +12,5 @@
     thunk->data.leaf.capt0.range.start_loc = chunk->pos_loc;
     thunk->data.leaf.capt0.range.end_loc = ctx->position_offset_loc;
 <%- end -%>
-    pcc_thunk_array__add(ctx->auxil, &chunk->thunks, thunk);
+    packcr_thunk_array__add(ctx->auxil, &chunk->thunks, thunk);
 }

data/lib/packcr/templates/node/action.rs.erb

@@ -0,0 +1,6 @@
+answer.push_leaf(
+    Action::<%= Packcr.camelize(gen.rule.name) %><%= index %>,
+    self.input.position_offset,
+    &[<% vars.each_with_index do |var, i| %><%= ", " if i > 0 %><%= var.index %><% end %>],
+    &[<% capts.each_with_index do |capt, i| %><%= ", " if i > 0 %><%= capt.index %><% end %>],
+);

data/lib/packcr/templates/node/alternate.c.erb

@@ -2,7 +2,7 @@
 {
     const size_t p = ctx->position_offset;
 <%- if gen.location -%>
-    const pcc_location_t p_loc = ctx->position_offset_loc;
+    const packcr_location_t p_loc = ctx->position_offset_loc;
 <%- end -%>
     const size_t n = chunk->thunks.len;
 <%- nodes.each_with_index do |expr, i| -%>
@@ -25,7 +25,7 @@ L<%= format("%04d", l) %>:;
 <%- if gen.location -%>
     ctx->position_offset_loc = p_loc;
 <%- end -%>
-    pcc_thunk_array__revert(ctx->auxil, &chunk->thunks, n);
+    packcr_thunk_array__revert(ctx->auxil, &chunk->thunks, n);
 <%-   if !c -%>
     goto L<%= format("%04d", onfail) %>;
 <%-   end -%>

data/lib/packcr/templates/node/alternate.rs.erb

@@ -0,0 +1,39 @@
+<%-   m = gen.next_label -%>
+'L<%= format("%04d", m) %>: {
+    let p = self.input.position_offset;
+    <%- if gen.location -%>
+    TODO
+    <%- end -%>
+    <%- nodes.each_with_index do |expr, i| -%>
+    {
+        <%-   c = i + 1 < nodes.length -%>
+        <%-   if expr.reversible?(gen) -%>
+        <%-     %><%= gen.generate_code(expr, m, 8, false, reverse: true, oncut: onfail) -%>
+        <%-   else -%>
+        <%-     l = gen.next_label -%>
+        'L<%= format("%04d", l) %>: {
+            <%- r = expr.reachability -%>
+            <%- %><%= gen.generate_code(expr, l, 12, false, oncut: onfail) -%>
+            <%- if r == Packcr::CODE_REACH__ALWAYS_SUCCEED -%>
+            <%-   if c -%>
+            // unreachable codes omitted
+            <%-   end -%>
+        }
+        <%-       break -%>
+        <%-     else -%>
+            <%-   if r == Packcr::CODE_REACH__BOTH -%>
+            break 'L<%= format("%04d", m) %>;
+            <%-   end -%>
+            <%- end -%>
+        }
+        <%- end -%>
+    }
+    self.input.position_offset = p;
+    <%- if gen.location -%>
+    TODO
+    <%- end -%>
+    <%-   if !c -%>
+    break 'L<%= format("%04d", onfail) %>;
+    <%-   end -%>
+    <%- end -%>
+} // 'L<%= format("%04d", m) %>

data/lib/packcr/templates/node/capture.c.erb

@@ -2,8 +2,8 @@
     const size_t p = ctx->position_offset;
     size_t q;
 <%- if gen.location -%>
-    pcc_location_t p_loc = ctx->position_offset_loc;
-    pcc_location_t q_loc;
+    packcr_location_t p_loc = ctx->position_offset_loc;
+    packcr_location_t q_loc;
 <%- end -%>
 <%= gen.generate_code(expr, onfail, 4, false) -%>
     q = ctx->position_offset;