packcr 0.0.8 → 0.1.1

data/lib/packcr/templates/context/source.rs.erb

@@ -0,0 +1,627 @@
+/* A packrat parser generated by PackCR <%= Packcr::VERSION %> */
+
+use std::cell::RefCell;
+use std::collections::{HashMap, HashSet};
+use std::io::Read;
+use std::rc::Rc;
+<%- if !code(:esource).empty? -%>
+
+<%-   code(:esource).each do |code| -%>
+<%=     stream.get_code_block(code, 0, @iname) -%>
+<%-   end -%>
+<%- end -%>
+<%- use_value = @root.rules.map {|r| r.vars.size }.max > 0 -%>
+
+struct LrMemoTable {
+    memos: HashMap<usize, LrMemoMap>,
+}
+impl LrMemoTable {
+    fn new() -> Self {
+        Self {
+            memos: HashMap::new(),
+        }
+    }
+
+    fn clear(&mut self) {
+        self.memos.clear();
+    }
+
+    fn set(&mut self, index: usize, rule: Rule, memo: LrMemo) {
+        let memo_map = self.memos.entry(index).or_default();
+        memo_map.insert(rule, memo);
+    }
+
+    fn get(&mut self, index: usize, rule: Rule) -> &mut LrMemo {
+        self.memos.get_mut(&index).unwrap().get_mut(&rule).unwrap()
+    }
+    fn has(&self, index: usize, rule: Rule) -> bool {
+        self.memos
+            .get(&index)
+            .is_some_and(|memo_map| memo_map.contains_key(&rule))
+    }
+}
+
+struct LrMemo {
+    offset: usize,
+    answer: Option<Rc<RefCell<ThunkChunk>>>,
+    grow: bool,
+}
+
+impl LrMemo {
+    fn new(offset: usize) -> Self {
+        Self {
+            offset,
+            answer: None,
+            grow: false,
+        }
+    }
+
+    fn start_grow(&mut self) -> bool {
+        if !self.grow && self.answer.is_none() {
+            self.grow = true;
+            true
+        } else {
+            false
+        }
+    }
+
+    fn update(&mut self, answer: Option<ThunkChunk>, offset: usize) {
+        self.answer = answer.map(|c| Rc::new(RefCell::new(c)));
+        self.offset = offset;
+    }
+
+    fn clone_answer(&mut self) -> Option<Rc<RefCell<ThunkChunk>>> {
+        self.answer.as_ref().map(Rc::clone)
+    }
+}
+
+struct ThunkChunk {
+    thunks: Rc<RefCell<Vec<Thunk>>>,
+    capts: CaptureTable,
+    pos: usize,
+    values: ValueTable,
+}
+
+impl ThunkChunk {
+    fn new(pos: usize) -> Self {
+        Self {
+            values: ValueTable::new(),
+            capts: CaptureTable::new(),
+            thunks: Rc::new(RefCell::new(Vec::new())),
+            pos,
+        }
+    }
+
+    fn push_leaf(
+        &self,
+        action: Action,
+        end: usize,
+        value_indices: &[usize],
+        capt_indices: &[usize],
+    ) {
+        {
+            let start = self.pos;
+            let mut value_refs = HashMap::new();
+            for &index in value_indices {
+                value_refs.insert(index, ValueRef::new(index, self.values.clone()));
+            }
+            let mut capts = HashMap::new();
+            for &index in capt_indices {
+                capts.insert(index, self.capts[index].clone());
+            }
+            let leaf = Thunk::Leaf(ThunkLeaf::new(action, start, end, value_refs, capts));
+            self.thunks.borrow_mut().push(leaf);
+        }
+    }
+
+    fn value_ref(&self, index: usize) -> ValueRef {
+        ValueRef::new(index, self.values.clone())
+    }
+}
+
+enum Thunk {
+    Leaf(ThunkLeaf),
+    Node(ThunkNode),
+}
+impl Thunk {
+    fn do_action(&self, processor: &ThunkProcessor, value: ValueRef) -> Value {
+        match self {
+            Thunk::Leaf(leaf) => leaf.do_action(processor, value),
+            Thunk::Node(node) => node.do_action(processor),
+        }
+    }
+}
+
+#[allow(dead_code)]
+struct ThunkLeaf {
+    value_refs: HashMap<usize, ValueRef>,
+    capts: HashMap<usize, Capture>,
+    capt0: Capture,
+    action: Action,
+}
+impl ThunkLeaf {
+    fn new(
+        action: Action,
+        start: usize,
+        end: usize,
+        value_refs: HashMap<usize, ValueRef>,
+        capts: HashMap<usize, Capture>,
+    ) -> Self {
+        Self {
+            value_refs,
+            capts,
+            capt0: Capture { start, end },
+            action,
+        }
+    }
+
+    fn do_action(&self, processor: &ThunkProcessor, mut value: ValueRef) -> Value {
+        value.with_mut(|v| {
+            processor.call_action(self.action, self, v);
+        });
+        value.get()
+    }
+    <%- if use_value -%>
+
+    fn values(&self) -> HashMap<usize, Value> {
+        self.value_refs.iter().map(|(k, v)| (*k, v.get())).collect()
+    }
+    <%- end -%>
+}
+
+struct ThunkNode {
+    thunks: Rc<RefCell<Vec<Thunk>>>,
+    value: ValueRef,
+}
+impl ThunkNode {
+    fn do_action(&self, processor: &ThunkProcessor) -> Value {
+        let mut v = 0;
+        for thunk in self.thunks.borrow_mut().iter_mut() {
+            v = thunk.do_action(processor, self.value.clone());
+        }
+        v
+    }
+}
+
+#[derive(Clone)]
+struct Capture {
+    start: usize,
+    end: usize,
+}
+
+type Value = i32;
+
+struct ValueTable {
+    table: Rc<RefCell<HashMap<usize, Value>>>,
+}
+impl ValueTable {
+    fn new() -> Self {
+        Self {
+            table: Rc::new(RefCell::new(HashMap::new())),
+        }
+    }
+    <%- if use_value -%>
+
+    fn clear(&mut self) {
+        self.table.borrow_mut().clear();
+    }
+    <%- end -%>
+
+    fn with_mut<F>(&mut self, index: usize, f: F)
+    where
+        F: FnOnce(&mut Value),
+    {
+        let mut table = self.table.borrow_mut();
+        let value = table.entry(index).or_insert(0);
+        f(value);
+    }
+}
+impl Clone for ValueTable {
+    fn clone(&self) -> Self {
+        Self {
+            table: self.table.clone(),
+        }
+    }
+}
+
+struct ValueRef {
+    values: ValueTable,
+    index: usize,
+}
+impl ValueRef {
+    fn new(index: usize, values: ValueTable) -> Self {
+        Self { index, values }
+    }
+
+    fn with_mut<F>(&mut self, f: F)
+    where
+        F: FnOnce(&mut i32),
+    {
+        self.values.with_mut(self.index, f);
+    }
+
+    fn get(&self) -> i32 {
+        *self.values.table.borrow().get(&self.index).unwrap_or(&0)
+    }
+}
+impl Clone for ValueRef {
+    fn clone(&self) -> Self {
+        Self {
+            index: self.index,
+            values: self.values.clone(),
+        }
+    }
+}
+
+struct CaptureTable {
+    table: HashMap<usize, Capture>,
+    len: usize,
+}
+
+type RuleSet = HashSet<Rule>;
+type RuleLimit = Option<RuleSet>;
+
+type LrMemoMap = HashMap<Rule, LrMemo>;
+
+impl CaptureTable {
+    fn new() -> Self {
+        Self {
+            table: HashMap::new(),
+            len: 0,
+        }
+    }
+
+    fn resize(&mut self, len: usize) {
+        let current_len = self.len;
+        if len > current_len {
+            for i in current_len..len {
+                self.table.insert(i, Capture { start: 0, end: 0 });
+            }
+        } else if len < current_len {
+            for i in len..current_len {
+                self.table.remove(&i);
+            }
+        }
+        self.len = len;
+    }
+}
+impl std::ops::Index<usize> for CaptureTable {
+    type Output = Capture;
+
+    fn index(&self, index: usize) -> &Self::Output {
+        &self.table[&index]
+    }
+}
+impl std::ops::IndexMut<usize> for CaptureTable {
+    fn index_mut(&mut self, index: usize) -> &mut Self::Output {
+        if self.len <= index {
+            self.resize(index + 1);
+        }
+        self.table.get_mut(&index).unwrap()
+    }
+}
+
+struct Input {
+    input: Box<dyn std::io::Read>,
+    start_position: usize,
+    position_offset: usize,
+    buffer: String,
+    closed: bool,
+}
+
+impl Input {
+    fn new(input: impl std::io::Read + 'static) -> Self {
+        Self {
+            input: Box::new(input),
+            start_position: 0,
+            position_offset: 0,
+            buffer: String::new(),
+            closed: false,
+        }
+    }
+
+    fn refill_buffer(&mut self, num: usize) -> usize {
+        let mut len = self.buffer.len();
+        if len >= self.position_offset + num {
+            return len - self.position_offset;
+        }
+
+        let mut input_buffer = [0u8; 1024];
+
+        while len < self.position_offset + num {
+            match self.input.read(&mut input_buffer) {
+                Ok(0) => break,
+                Ok(bytes_read) => {
+                    let s = std::string::String::from_utf8_lossy(&input_buffer[..bytes_read]);
+                    self.buffer.push_str(&s);
+                }
+                Err(_) => break,
+            }
+            len = self.buffer.len();
+        }
+
+        len - self.position_offset
+    }
+
+    fn commit_buffer(&mut self) {
+        let position_offset = self.position_offset;
+
+        self.buffer.drain(..position_offset);
+
+        self.start_position += position_offset;
+        self.position_offset = 0;
+    }
+
+    #[allow(dead_code)]
+    fn get_char_as_utf32(&mut self) -> (i32, usize) {
+        if self.refill_buffer(1) < 1 {
+            return (0, 0);
+        }
+
+        let current_position = self.position_offset;
+        let remaining_chars: &str = &self.buffer[current_position..];
+
+        if let Some(ch) = remaining_chars.chars().next() {
+            let bytes_used = ch.len_utf8();
+
+            if self.refill_buffer(bytes_used) < bytes_used {
+                return (0, 0);
+            }
+
+            let utf32_value = ch as u32 as i32;
+
+            return (utf32_value, bytes_used);
+        }
+
+        (0, 0)
+    }
+
+    fn back_to(&mut self, memo: &mut LrMemo) -> Option<Rc<RefCell<ThunkChunk>>> {
+        self.position_offset = memo.offset;
+        memo.clone_answer()
+    }
+}
+
+const NOP: Result<usize, usize> = Ok(0);
+const fn throw(label: usize) -> Result<usize, usize> {
+    Err(label)
+}
+
+fn catch(label: usize, f: impl FnOnce() -> Result<usize, usize>) -> Result<usize, usize> {
+    match f() {
+        Ok(_) => NOP,
+        Err(e) if e == label => NOP,
+        Err(e) => throw(e),
+    }
+}
+
+struct <%= class_name %> {
+    level: usize,
+    memos: LrMemoTable,
+    input: Input,
+}
+
+impl <%= class_name %> {
+    fn new(input: impl std::io::Read + 'static) -> Self {
+        Self {
+            level: 0,
+            memos: LrMemoTable::new(),
+            input: Input::new(input),
+        }
+    }
+
+    fn parse(&mut self) -> Option<Value> {
+        if self.input.closed {
+            return None;
+        }
+
+        let mut answer = ThunkChunk::new(0);
+        let pos = self.input.start_position;
+        <%- if !@root.rules.empty? -%>
+
+        let value = {
+            let node = self.rule_thunk_node(
+                Rule::<%= Packcr.camelize(@root.rules[0].name) %>,
+                &mut answer,
+                0,
+                self.input.position_offset,
+                &None,
+            );
+            self.memos.clear();
+            node.map(|node| node.do_action(&ThunkProcessor::new(&self.input.buffer)))
+        };
+        self.input.commit_buffer();
+        if pos == self.input.start_position || self.input.refill_buffer(1) < 1 {
+            self.input.closed = true;
+        }
+        value
+        <%- end -%>
+    }
+
+    fn grow_lr(&mut self, rule: Rule, offset: usize) {
+        loop {
+            let old_offset = self.input.position_offset;
+            self.input.position_offset = offset;
+
+            let answer = self.call_rule(rule, offset, Some(RuleSet::new()));
+            match answer {
+                Some(answer) if self.input.position_offset > old_offset => {
+                    let memo = self.memos.get(offset, rule);
+                    memo.update(Some(answer), self.input.position_offset);
+                }
+                _ => {
+                    return;
+                }
+            }
+        }
+    }
+
+    fn rule_answer_without_limits(&mut self, rule: Rule) -> Option<Rc<RefCell<ThunkChunk>>> {
+        let offset = self.input.position_offset;
+        if !self.memos.has(offset, rule) {
+            let memo = LrMemo::new(offset);
+            self.memos.set(offset, rule, memo);
+            let answer = self.call_rule(rule, offset, None);
+
+            let memo = self.memos.get(offset, rule);
+            memo.update(answer, self.input.position_offset);
+            if !memo.grow {
+                return self.input.back_to(memo);
+            }
+            self.grow_lr(rule, offset);
+
+            let memo = self.memos.get(offset, rule);
+            memo.grow = false;
+            return self.input.back_to(memo);
+        }
+
+        let memo = self.memos.get(offset, rule);
+        if memo.start_grow() {
+            return None;
+        }
+        self.input.back_to(memo)
+    }
+
+    fn rule_answer_with_limits(
+        &mut self,
+        rule: Rule,
+        limits: RuleSet,
+    ) -> Option<Rc<RefCell<ThunkChunk>>> {
+        let offset = self.input.position_offset;
+        let answer = self.call_rule(rule, offset, Some(limits));
+        if !self.memos.has(offset, rule) {
+            return answer.map(|answer| Rc::new(RefCell::new(answer)));
+        }
+
+        let memo = self.memos.get(offset, rule);
+        if self.input.position_offset > memo.offset {
+            memo.update(answer, self.input.position_offset);
+        }
+        self.input.back_to(memo)
+    }
+
+    fn apply_rule(
+        &mut self,
+        rule: Rule,
+        parent: &mut ThunkChunk,
+        value_index: usize,
+        offset: usize,
+        limits: &RuleLimit,
+    ) -> bool {
+        let node = self.rule_thunk_node(rule, parent, value_index, offset, limits);
+        node.is_some_and(|node| {
+            parent.thunks.borrow_mut().push(Thunk::Node(node));
+            true
+        })
+    }
+
+    fn rule_thunk_node(
+        &mut self,
+        rule: Rule,
+        parent: &ThunkChunk,
+        value_index: usize,
+        offset: usize,
+        limits: &RuleLimit,
+    ) -> Option<ThunkNode> {
+        match limits {
+            Some(limit_set)
+                if self.input.position_offset == offset && !limit_set.contains(&rule) =>
+            {
+                self.rule_answer_with_limits(rule, limits.clone().unwrap())
+            }
+            _ => self.rule_answer_without_limits(rule),
+        }
+        .map(|answer| ThunkNode {
+            thunks: answer.borrow().thunks.clone(),
+            value: parent.value_ref(value_index),
+        })
+    }
+
+    fn call_rule(
+        &mut self,
+        rule: Rule,
+        offset: usize,
+        mut limits: RuleLimit,
+    ) -> Option<ThunkChunk> {
+        if let Some(ref mut limits) = limits {
+            limits.insert(rule);
+        }
+        match rule {
+            <%- @root.rules.each do |rule| -%>
+            Rule::<%= Packcr.camelize(rule.name) %> => self.evaluate_rule_<%= rule.name %>(offset, limits),
+            <%- end -%>
+        }
+    }
+    <%- @root.rules.each do |rule| -%>
+
+    <%-   gen = ::Packcr::Generator.new(rule, @ascii, @location, :rs) -%>
+    <%- %><%= gen.generate_code(rule, 0, 4, false) -%>
+    <%- end -%>
+}
+
+struct ThunkProcessor<'a> {
+    buffer: &'a str,
+}
+
+impl<'a> ThunkProcessor<'a> {
+    fn new(buffer: &'a str) -> Self {
+        Self { buffer }
+    }
+
+    fn call_action(&self, action: Action, thunk: &ThunkLeaf, value: &mut i32) {
+        match action {
+            <%- @root.rules.each do |rule| -%>
+            <%-   rule.codes.each do |code| -%>
+            Action::<%= Packcr.camelize(rule.name) %><%= code.index %> => self.action_<%= rule.name %>_<%= code.index %>(thunk, value),
+            <%-   end -%>
+            <%- end -%>
+        }
+    }
+    <%- @root.rules.each do |rule| -%>
+    <%-   rule.codes.each do |code| -%>
+
+    #[allow(unused_variables, non_snake_case)]
+    fn action_<%= rule.name %>_<%= code.index %>(&self, leaf: &ThunkLeaf, out: &mut i32) {
+        <%-   code.vars.each_with_index do |ref, i| -%>
+        <%-     if i == 0 -%>
+        let values = leaf.values();
+        <%-     end -%>
+        let <%= ref.var %> = values[&<%= ref.index %>];
+        <%-   end -%>
+        let _0 = {
+            let capt = &leaf.capt0;
+            &self.buffer[(capt.start)..(capt.end)]
+        };
+        <%-   code.capts.size.times do |i| -%>
+        let _<%= i + 1 %> = {
+            let capt = &leaf.capts[&<%= i %>];
+            &self.buffer[(capt.start)..(capt.end)]
+        };
+        <%-   end -%>
+    <%=   stream.get_code_block(code.code, 4, @iname) -%>
+    }
+    <%-   end -%>
+    <%- end -%>
+}
+
+#[derive(Copy, Clone)]
+enum Action {
+    <%- @root.rules.each do |rule| -%>
+    <%-   rule.codes.each do |code| -%>
+    <%= Packcr.camelize(rule.name) %><%= code.index %>,
+    <%-   end -%>
+    <%- end -%>
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash)]
+enum Rule {
+  <%- @root.rules.each do |rule| -%>
+    <%= Packcr.camelize(rule.name) %>,
+  <%- end -%>
+}
+<%- if !code(:lsource).empty? -%>
+
+<%-   code(:lsource).each do |code| -%>
+<%=     stream.get_code_block(code, 0, @iname) -%>
+<%-   end -%>
+<%- end -%>

data/lib/packcr/templates/node/action.rs.erb

@@ -0,0 +1,6 @@
+answer.push_leaf(<% -%>
+    <%- %>Action::<%= Packcr.camelize(gen.rule.name) %><%= index %>, <% -%>
+    <%- %>self.input.position_offset, <% -%>
+    <%- %>&[<% vars.each_with_index do |var, i| %><%= ", " if i > 0 %><%= var.index %><% end %>], <% -%>
+    <%- %>&[<% capts.each_with_index do |capt, i| %><%= ", " if i > 0 %><%= capt.index %><% end %>]<% -%>
+);

data/lib/packcr/templates/node/alternate.rs.erb

@@ -0,0 +1,38 @@
+<%- m = gen.next_label -%>
+catch(<%= m %>, || {
+    let p = self.input.position_offset;
+    <%- if gen.location -%>
+    TODO
+    <%- end -%>
+    <%- nodes.each_with_index do |expr, i| -%>
+    <%-   c = i + 1 < nodes.length -%>
+    <%-   if expr.reversible?(gen) -%>
+    <%-     %><%= gen.generate_code(expr, m, 4, false, reverse: true, oncut: onfail) -%>
+    <%-   else -%>
+    <%-     l = gen.next_label -%>
+    <%-     r = expr.reachability -%>
+    <%-     if r == Packcr::CODE_REACH__ALWAYS_SUCCEED -%>
+    catch(<%= l %>, || {
+        <%-   %><%= gen.generate_code(expr, l, 8, false, oncut: onfail) -%>
+        <%-   if c -%>
+        // unreachable codes omitted
+        <%-   end -%>
+        NOP
+    })?
+    <%-       break -%>
+    <%-     else -%>
+    catch(<%= l %>, || {
+        <%-   %><%= gen.generate_code(expr, l, 8, false, oncut: onfail) -%>
+        throw(<%= m %>)
+    })?;
+    <%-     end -%>
+    <%-   end -%>
+    self.input.position_offset = p;
+    <%-   if gen.location -%>
+    TODO
+    <%-   end -%>
+    <%-   if !c -%>
+    throw(<%= onfail %>)
+    <%-   end -%>
+    <%- end -%>
+})?;

data/lib/packcr/templates/node/capture.rs.erb

@@ -0,0 +1,13 @@
+let p_inner = self.input.position_offset;
+<%- if gen.location -%>
+TODO
+<%- end -%>
+{
+    <%- %><%= gen.generate_code(expr, onfail, 4, false) -%>
+}
+let q = self.input.position_offset;
+answer.capts[<%= index %>].start = p_inner;
+answer.capts[<%= index %>].end = q;
+<%- if gen.location -%>
+TODO
+<%- end -%>