kumi-parser 0.0.9 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CLAUDE.md +2 -2
- data/examples/debug_text_parser.rb +1 -1
- data/examples/text_parser_comprehensive_test.rb +1 -1
- data/examples/text_parser_test_with_comments.rb +2 -2
- data/kumi-parser.gemspec +1 -1
- data/lib/kumi/parser/direct_parser.rb +36 -20
- data/lib/kumi/parser/smart_tokenizer.rb +12 -0
- data/lib/kumi/parser/token_metadata.rb +13 -1
- data/lib/kumi/parser/version.rb +1 -1
- metadata +8 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3e8f52b91ed0f66c7bdc49935483a81a5f329cec5ef5914555a57c6200b48ad7
|
4
|
+
data.tar.gz: 3b52cc1430c0924cfc4a88f7facb2f022f51ef91ffdc94441b073d2dbde1866e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 28dbc7918ed27110f6922286d36151ee00867a2a3d2afb8b20895fe12ddff5c3695b7dfa17058ce6a3638ddb43ca673661c85e586f31a6be4e94a7ea6364a1d7
|
7
|
+
data.tar.gz: 74e0285aa3c6df1b0a41dd6ace23f60421202835f3d864089bcbc6fa6594dc6f0bc25b4db429cfff2880b633e63836e8b402195ce5db2e9d312cf5c622688fbf
|
data/CLAUDE.md
CHANGED
@@ -25,7 +25,7 @@
|
|
25
25
|
## AST Structure & Compatibility
|
26
26
|
|
27
27
|
All nodes from `Kumi::Syntax::*` (defined in main kumi gem):
|
28
|
-
- `Root(inputs,
|
28
|
+
- `Root(inputs, values, traits)`
|
29
29
|
- `InputDeclaration(name, domain, type, children)`
|
30
30
|
- `ValueDeclaration(name, expression)`
|
31
31
|
- `TraitDeclaration(name, expression)`
|
@@ -50,7 +50,7 @@ ast = Kumi::Parser::TextParser.parse(schema)
|
|
50
50
|
puts Kumi::Support::SExpressionPrinter.print(ast)
|
51
51
|
# => (Root
|
52
52
|
# inputs: [(InputDeclaration :income :float)]
|
53
|
-
#
|
53
|
+
# values: [(ValueDeclaration :tax (CallExpression :+ ...))]
|
54
54
|
# traits: [(TraitDeclaration :adult (CallExpression :>= ...))])
|
55
55
|
```
|
56
56
|
|
@@ -33,7 +33,7 @@ begin
|
|
33
33
|
puts
|
34
34
|
|
35
35
|
puts 'AST structure:'
|
36
|
-
puts "- Values: #{ast.
|
36
|
+
puts "- Values: #{ast.values.count} - #{ast.values.map(&:name)}"
|
37
37
|
puts "- Traits: #{ast.traits.count} - #{ast.traits.map(&:name)}"
|
38
38
|
rescue StandardError => e
|
39
39
|
puts "Error: #{e.message}"
|
@@ -318,7 +318,7 @@ if __FILE__ == $0
|
|
318
318
|
ast = Kumi::TextParser.parse(schema_text)
|
319
319
|
puts "\nParsed successfully!"
|
320
320
|
puts "- Input fields: #{ast.inputs.map(&:name).join(', ')}"
|
321
|
-
puts "- Values: #{ast.
|
321
|
+
puts "- Values: #{ast.values.count}"
|
322
322
|
puts "- Traits: #{ast.traits.count}"
|
323
323
|
else
|
324
324
|
puts '❌ Schema has errors:'
|
@@ -70,8 +70,8 @@ begin
|
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
73
|
-
puts "- Value declarations: #{ast.
|
74
|
-
ast.
|
73
|
+
puts "- Value declarations: #{ast.values.count}"
|
74
|
+
ast.values.each_with_index do |value, i|
|
75
75
|
puts " #{i + 1}. #{value.name}"
|
76
76
|
end
|
77
77
|
|
data/kumi-parser.gemspec
CHANGED
@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
|
|
32
32
|
spec.require_paths = ['lib']
|
33
33
|
|
34
34
|
# Dependencies
|
35
|
-
spec.add_dependency 'kumi', '~> 0.0.
|
35
|
+
spec.add_dependency 'kumi', '~> 0.0.13'
|
36
36
|
spec.add_dependency 'parslet', '~> 2.0'
|
37
37
|
spec.add_dependency 'zeitwerk', '~> 2.6'
|
38
38
|
|
@@ -75,7 +75,7 @@ module Kumi
|
|
75
75
|
# Construct Root with exact AST.md structure
|
76
76
|
Kumi::Syntax::Root.new(
|
77
77
|
input_declarations,
|
78
|
-
value_declarations, #
|
78
|
+
value_declarations, # values
|
79
79
|
trait_declarations,
|
80
80
|
loc: schema_token.location
|
81
81
|
)
|
@@ -126,9 +126,9 @@ module Kumi
|
|
126
126
|
end
|
127
127
|
end
|
128
128
|
|
129
|
-
# Handle nested array declarations
|
129
|
+
# Handle nested array and hash declarations
|
130
130
|
children = []
|
131
|
-
if type_token.metadata[:type_name]
|
131
|
+
if %i[array hash].include?(type_token.metadata[:type_name]) && current_token.type == :do
|
132
132
|
advance # consume 'do'
|
133
133
|
skip_comments_and_newlines
|
134
134
|
|
@@ -143,13 +143,24 @@ module Kumi
|
|
143
143
|
expect_token(:end)
|
144
144
|
end
|
145
145
|
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
146
|
+
if children.empty?
|
147
|
+
Kumi::Syntax::InputDeclaration.new(
|
148
|
+
name_token.value,
|
149
|
+
domain,
|
150
|
+
type_token.metadata[:type_name],
|
151
|
+
children,
|
152
|
+
loc: type_token.location
|
153
|
+
)
|
154
|
+
else
|
155
|
+
Kumi::Syntax::InputDeclaration.new(
|
156
|
+
name_token.value,
|
157
|
+
domain,
|
158
|
+
type_token.metadata[:type_name],
|
159
|
+
children,
|
160
|
+
:field,
|
161
|
+
loc: type_token.location
|
162
|
+
)
|
163
|
+
end
|
153
164
|
end
|
154
165
|
|
155
166
|
def parse_domain_specification
|
@@ -175,7 +186,7 @@ module Kumi
|
|
175
186
|
start_token = current_token
|
176
187
|
start_value = start_token.type == :integer ? start_token.value.to_i : start_token.value.to_f
|
177
188
|
advance
|
178
|
-
|
189
|
+
|
179
190
|
case current_token.type
|
180
191
|
when :dot_dot
|
181
192
|
# Inclusive range: start..end
|
@@ -199,14 +210,14 @@ module Kumi
|
|
199
210
|
|
200
211
|
def convert_array_expression_to_ruby_array(array_expr)
|
201
212
|
return nil unless array_expr.is_a?(Kumi::Syntax::ArrayExpression)
|
202
|
-
|
213
|
+
|
203
214
|
array_expr.elements.map do |element|
|
204
215
|
if element.is_a?(Kumi::Syntax::Literal)
|
205
216
|
element.value
|
206
217
|
else
|
207
218
|
# For non-literal elements, we'd need more complex evaluation
|
208
219
|
# For now, just return the element as-is
|
209
|
-
element
|
220
|
+
element
|
210
221
|
end
|
211
222
|
end
|
212
223
|
end
|
@@ -268,10 +279,10 @@ module Kumi
|
|
268
279
|
when :on
|
269
280
|
on_token = advance_and_return_token
|
270
281
|
condition = parse_expression
|
271
|
-
|
272
|
-
# Wrap simple trait references in
|
282
|
+
|
283
|
+
# Wrap simple trait references in cascade_and to match Ruby DSL behavior
|
273
284
|
condition = wrap_condition_in_all(condition) if simple_trait_reference?(condition)
|
274
|
-
|
285
|
+
|
275
286
|
expect_token(:comma)
|
276
287
|
result = parse_expression
|
277
288
|
|
@@ -335,7 +346,7 @@ module Kumi
|
|
335
346
|
token = current_token
|
336
347
|
|
337
348
|
case token.type
|
338
|
-
when :integer, :float, :string, :boolean
|
349
|
+
when :integer, :float, :string, :boolean, :constant
|
339
350
|
# Direct AST construction using token metadata
|
340
351
|
value = convert_literal_value(token)
|
341
352
|
advance
|
@@ -529,6 +540,12 @@ module Kumi
|
|
529
540
|
when :float then token.value.gsub('_', '').to_f
|
530
541
|
when :string then token.value
|
531
542
|
when :boolean then token.value == 'true'
|
543
|
+
when :constant
|
544
|
+
case token.value
|
545
|
+
when 'Float::INFINITY' then Float::INFINITY
|
546
|
+
else
|
547
|
+
raise_parse_error("Unknown constant: #{token.value}")
|
548
|
+
end
|
532
549
|
end
|
533
550
|
end
|
534
551
|
|
@@ -553,10 +570,9 @@ module Kumi
|
|
553
570
|
condition.is_a?(Kumi::Syntax::DeclarationReference)
|
554
571
|
end
|
555
572
|
|
556
|
-
# Helper method to wrap condition in
|
573
|
+
# Helper method to wrap condition in cascade_and function call
|
557
574
|
def wrap_condition_in_all(condition)
|
558
|
-
|
559
|
-
Kumi::Syntax::CallExpression.new(:all?, [array_expr], loc: condition.loc)
|
575
|
+
Kumi::Syntax::CallExpression.new(:cascade_and, [condition], loc: condition.loc)
|
560
576
|
end
|
561
577
|
|
562
578
|
# Map operator token types to function names for Ruby DSL compatibility
|
@@ -160,6 +160,18 @@ module Kumi
|
|
160
160
|
start_column = @column
|
161
161
|
identifier = consume_while { |c| c.match?(/[a-zA-Z0-9_]/) }
|
162
162
|
|
163
|
+
# Check if it's a constant (e.g., Float::INFINITY)
|
164
|
+
if identifier == 'Float' && current_char == ':' && peek_char == ':'
|
165
|
+
advance # consume first :
|
166
|
+
advance # consume second :
|
167
|
+
constant_name = consume_while { |c| c.match?(/[a-zA-Z0-9_]/) }
|
168
|
+
full_constant = "#{identifier}::#{constant_name}"
|
169
|
+
|
170
|
+
location = Kumi::Syntax::Location.new(file: @source_file, line: @line, column: start_column)
|
171
|
+
@tokens << Token.new(:constant, full_constant, location, Kumi::Parser::TOKEN_METADATA[:constant])
|
172
|
+
return
|
173
|
+
end
|
174
|
+
|
163
175
|
# Check if it's a keyword
|
164
176
|
if keyword_type = Kumi::Parser::KEYWORDS[identifier]
|
165
177
|
metadata = Kumi::Parser::TOKEN_METADATA[keyword_type].dup
|
@@ -13,6 +13,7 @@ module Kumi
|
|
13
13
|
# Identifiers and symbols
|
14
14
|
IDENTIFIER = :identifier
|
15
15
|
SYMBOL = :symbol # :name
|
16
|
+
CONSTANT = :constant # Float::INFINITY
|
16
17
|
|
17
18
|
# Keywords
|
18
19
|
SCHEMA = :schema
|
@@ -143,6 +144,11 @@ module Kumi
|
|
143
144
|
starts_declaration: true,
|
144
145
|
type_name: :array
|
145
146
|
},
|
147
|
+
hash_type: {
|
148
|
+
category: :type_keyword,
|
149
|
+
starts_declaration: true,
|
150
|
+
type_name: :hash
|
151
|
+
},
|
146
152
|
|
147
153
|
# Function keyword
|
148
154
|
fn: {
|
@@ -278,6 +284,11 @@ module Kumi
|
|
278
284
|
starts_expression: true,
|
279
285
|
is_declaration_name: true
|
280
286
|
},
|
287
|
+
constant: {
|
288
|
+
category: :literal,
|
289
|
+
starts_expression: true,
|
290
|
+
ast_class: 'Kumi::Syntax::Literal'
|
291
|
+
},
|
281
292
|
|
282
293
|
# Punctuation with parser hints
|
283
294
|
dot: {
|
@@ -373,7 +384,8 @@ module Kumi
|
|
373
384
|
'string' => :string_type,
|
374
385
|
'boolean' => :boolean_type,
|
375
386
|
'any' => :any_type,
|
376
|
-
'array' => :array_type
|
387
|
+
'array' => :array_type,
|
388
|
+
'hash' => :hash_type
|
377
389
|
}.freeze
|
378
390
|
|
379
391
|
# Opener to closer mappings for error recovery
|
data/lib/kumi/parser/version.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kumi-parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kumi Team
|
8
|
+
autorequire:
|
8
9
|
bindir: exe
|
9
10
|
cert_chain: []
|
10
|
-
date:
|
11
|
+
date: 2025-08-21 00:00:00.000000000 Z
|
11
12
|
dependencies:
|
12
13
|
- !ruby/object:Gem::Dependency
|
13
14
|
name: kumi
|
@@ -15,14 +16,14 @@ dependencies:
|
|
15
16
|
requirements:
|
16
17
|
- - "~>"
|
17
18
|
- !ruby/object:Gem::Version
|
18
|
-
version: 0.0.
|
19
|
+
version: 0.0.13
|
19
20
|
type: :runtime
|
20
21
|
prerelease: false
|
21
22
|
version_requirements: !ruby/object:Gem::Requirement
|
22
23
|
requirements:
|
23
24
|
- - "~>"
|
24
25
|
- !ruby/object:Gem::Version
|
25
|
-
version: 0.0.
|
26
|
+
version: 0.0.13
|
26
27
|
- !ruby/object:Gem::Dependency
|
27
28
|
name: parslet
|
28
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -161,6 +162,7 @@ metadata:
|
|
161
162
|
homepage_uri: https://github.com/amuta/kumi-parser
|
162
163
|
source_code_uri: https://github.com/amuta/kumi-parser
|
163
164
|
changelog_uri: https://github.com/amuta/kumi-parser/blob/main/CHANGELOG.md
|
165
|
+
post_install_message:
|
164
166
|
rdoc_options: []
|
165
167
|
require_paths:
|
166
168
|
- lib
|
@@ -175,7 +177,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
175
177
|
- !ruby/object:Gem::Version
|
176
178
|
version: '0'
|
177
179
|
requirements: []
|
178
|
-
rubygems_version: 3.
|
180
|
+
rubygems_version: 3.5.22
|
181
|
+
signing_key:
|
179
182
|
specification_version: 4
|
180
183
|
summary: Text parser for Kumi
|
181
184
|
test_files: []
|