mt-lang 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (258) hide show
  1. checksums.yaml +7 -0
  2. data/.ruby-version +1 -0
  3. data/AUTHORS +3 -0
  4. data/Gemfile +19 -0
  5. data/Gemfile.lock +82 -0
  6. data/LICENSE +21 -0
  7. data/README.md +1208 -0
  8. data/Rakefile +332 -0
  9. data/bin/mtc +8 -0
  10. data/bin/profile-mtc-checks +133 -0
  11. data/bin/tracy-profiler +0 -0
  12. data/docs/build-guide.md +536 -0
  13. data/docs/index.html +2778 -0
  14. data/docs/language-design.md +1519 -0
  15. data/docs/language-manual.md +1534 -0
  16. data/lib/milk_tea/base.rb +49 -0
  17. data/lib/milk_tea/bindings/bindgen/ast_parser.rb +398 -0
  18. data/lib/milk_tea/bindings/bindgen/declaration.rb +319 -0
  19. data/lib/milk_tea/bindings/bindgen/emitter.rb +387 -0
  20. data/lib/milk_tea/bindings/bindgen/overrides.rb +134 -0
  21. data/lib/milk_tea/bindings/bindgen/type_mapper.rb +622 -0
  22. data/lib/milk_tea/bindings/bindgen.rb +207 -0
  23. data/lib/milk_tea/bindings/cli.rb +121 -0
  24. data/lib/milk_tea/bindings/imported_bindings/defaults.rb +210 -0
  25. data/lib/milk_tea/bindings/imported_bindings/generator.rb +1114 -0
  26. data/lib/milk_tea/bindings/imported_bindings/method_source.rb +190 -0
  27. data/lib/milk_tea/bindings/imported_bindings/naming.rb +286 -0
  28. data/lib/milk_tea/bindings/imported_bindings.rb +215 -0
  29. data/lib/milk_tea/bindings/opengl_registry.rb +547 -0
  30. data/lib/milk_tea/bindings/raw_bindings/defaults.rb +1338 -0
  31. data/lib/milk_tea/bindings/raw_bindings.rb +269 -0
  32. data/lib/milk_tea/bindings/steamworks.rb +629 -0
  33. data/lib/milk_tea/bindings/upstream_sources.rb +352 -0
  34. data/lib/milk_tea/bindings/vendored_box2d.rb +79 -0
  35. data/lib/milk_tea/bindings/vendored_c_library.rb +322 -0
  36. data/lib/milk_tea/bindings/vendored_cjson.rb +52 -0
  37. data/lib/milk_tea/bindings/vendored_flecs.rb +78 -0
  38. data/lib/milk_tea/bindings/vendored_glfw.rb +77 -0
  39. data/lib/milk_tea/bindings/vendored_libuv.rb +75 -0
  40. data/lib/milk_tea/bindings/vendored_pcre2.rb +84 -0
  41. data/lib/milk_tea/bindings/vendored_raylib.rb +131 -0
  42. data/lib/milk_tea/bindings/vendored_sdl3.rb +78 -0
  43. data/lib/milk_tea/bindings/vendored_steamworks.rb +71 -0
  44. data/lib/milk_tea/bindings/vendored_tool.rb +71 -0
  45. data/lib/milk_tea/bindings/vendored_tools.rb +23 -0
  46. data/lib/milk_tea/bindings/vendored_tracy.rb +37 -0
  47. data/lib/milk_tea/bindings.rb +23 -0
  48. data/lib/milk_tea/core/ast.rb +311 -0
  49. data/lib/milk_tea/core/async_runtime_installer.rb +28 -0
  50. data/lib/milk_tea/core/binding_types.rb +18 -0
  51. data/lib/milk_tea/core/bindings/attribute_binding.rb +58 -0
  52. data/lib/milk_tea/core/bindings/function_binding.rb +5 -0
  53. data/lib/milk_tea/core/bindings/module_binding.rb +58 -0
  54. data/lib/milk_tea/core/bindings/value_binding.rb +21 -0
  55. data/lib/milk_tea/core/c_backend/aggregate_utils.rb +103 -0
  56. data/lib/milk_tea/core/c_backend/control_flow_emission.rb +412 -0
  57. data/lib/milk_tea/core/c_backend/expressions.rb +468 -0
  58. data/lib/milk_tea/core/c_backend/feature_detection.rb +483 -0
  59. data/lib/milk_tea/core/c_backend/reachability.rb +398 -0
  60. data/lib/milk_tea/core/c_backend/reinterpret.rb +226 -0
  61. data/lib/milk_tea/core/c_backend/runtime_helpers.rb +1080 -0
  62. data/lib/milk_tea/core/c_backend/statements.rb +563 -0
  63. data/lib/milk_tea/core/c_backend/type_collectors.rb +1545 -0
  64. data/lib/milk_tea/core/c_backend/type_declaration.rb +223 -0
  65. data/lib/milk_tea/core/c_backend/type_system.rb +345 -0
  66. data/lib/milk_tea/core/c_backend.rb +287 -0
  67. data/lib/milk_tea/core/compatibility_helpers.rb +79 -0
  68. data/lib/milk_tea/core/compile_time/const_eval.rb +187 -0
  69. data/lib/milk_tea/core/compile_time.rb +329 -0
  70. data/lib/milk_tea/core/control_flow/builder.rb +582 -0
  71. data/lib/milk_tea/core/control_flow/constant_propagation.rb +143 -0
  72. data/lib/milk_tea/core/control_flow/dataflow.rb +74 -0
  73. data/lib/milk_tea/core/control_flow/definite_assignment.rb +79 -0
  74. data/lib/milk_tea/core/control_flow/graph.rb +90 -0
  75. data/lib/milk_tea/core/control_flow/liveness.rb +24 -0
  76. data/lib/milk_tea/core/control_flow/nullability_flow.rb +43 -0
  77. data/lib/milk_tea/core/control_flow/reachability.rb +22 -0
  78. data/lib/milk_tea/core/control_flow/termination.rb +31 -0
  79. data/lib/milk_tea/core/control_flow.rb +34 -0
  80. data/lib/milk_tea/core/cst.rb +48 -0
  81. data/lib/milk_tea/core/cst_builder.rb +19 -0
  82. data/lib/milk_tea/core/ir.rb +85 -0
  83. data/lib/milk_tea/core/keywords.rb +97 -0
  84. data/lib/milk_tea/core/lexer/character_classes.rb +60 -0
  85. data/lib/milk_tea/core/lexer/format_strings.rb +225 -0
  86. data/lib/milk_tea/core/lexer/heredocs.rb +199 -0
  87. data/lib/milk_tea/core/lexer/indentation.rb +62 -0
  88. data/lib/milk_tea/core/lexer/numbers.rb +96 -0
  89. data/lib/milk_tea/core/lexer/recovery.rb +32 -0
  90. data/lib/milk_tea/core/lexer/strings.rb +167 -0
  91. data/lib/milk_tea/core/lexer/symbols.rb +71 -0
  92. data/lib/milk_tea/core/lexer/trivia.rb +53 -0
  93. data/lib/milk_tea/core/lexer.rb +430 -0
  94. data/lib/milk_tea/core/lowering/artifacts.rb +26 -0
  95. data/lib/milk_tea/core/lowering/async/analysis.rb +245 -0
  96. data/lib/milk_tea/core/lowering/async/lowering.rb +1399 -0
  97. data/lib/milk_tea/core/lowering/async/normalization.rb +459 -0
  98. data/lib/milk_tea/core/lowering/async.rb +714 -0
  99. data/lib/milk_tea/core/lowering/block.rb +1052 -0
  100. data/lib/milk_tea/core/lowering/calls.rb +1565 -0
  101. data/lib/milk_tea/core/lowering/declarations.rb +214 -0
  102. data/lib/milk_tea/core/lowering/dyn.rb +206 -0
  103. data/lib/milk_tea/core/lowering/events.rb +1054 -0
  104. data/lib/milk_tea/core/lowering/expressions.rb +1645 -0
  105. data/lib/milk_tea/core/lowering/foreign_cstr.rb +206 -0
  106. data/lib/milk_tea/core/lowering/functions.rb +242 -0
  107. data/lib/milk_tea/core/lowering/loops.rb +1087 -0
  108. data/lib/milk_tea/core/lowering/lowering_context.rb +80 -0
  109. data/lib/milk_tea/core/lowering/proc.rb +419 -0
  110. data/lib/milk_tea/core/lowering/resolve.rb +2516 -0
  111. data/lib/milk_tea/core/lowering/scans.rb +220 -0
  112. data/lib/milk_tea/core/lowering/str_buffer.rb +125 -0
  113. data/lib/milk_tea/core/lowering/utils.rb +1453 -0
  114. data/lib/milk_tea/core/lowering.rb +378 -0
  115. data/lib/milk_tea/core/module_binder.rb +181 -0
  116. data/lib/milk_tea/core/module_loader/errors.rb +21 -0
  117. data/lib/milk_tea/core/module_loader.rb +479 -0
  118. data/lib/milk_tea/core/module_path_resolver.rb +153 -0
  119. data/lib/milk_tea/core/module_roots.rb +88 -0
  120. data/lib/milk_tea/core/parser/attributes.rb +71 -0
  121. data/lib/milk_tea/core/parser/blocks.rb +119 -0
  122. data/lib/milk_tea/core/parser/declarations.rb +749 -0
  123. data/lib/milk_tea/core/parser/expressions.rb +624 -0
  124. data/lib/milk_tea/core/parser/recovery.rb +131 -0
  125. data/lib/milk_tea/core/parser/statements.rb +756 -0
  126. data/lib/milk_tea/core/parser/types.rb +271 -0
  127. data/lib/milk_tea/core/parser.rb +400 -0
  128. data/lib/milk_tea/core/prelude_installer.rb +29 -0
  129. data/lib/milk_tea/core/pretty_printer/ast_formatter.rb +917 -0
  130. data/lib/milk_tea/core/pretty_printer/base_formatter.rb +87 -0
  131. data/lib/milk_tea/core/pretty_printer/ir_formatter.rb +300 -0
  132. data/lib/milk_tea/core/pretty_printer.rb +17 -0
  133. data/lib/milk_tea/core/semantic_analyzer/analysis_context.rb +314 -0
  134. data/lib/milk_tea/core/semantic_analyzer/attributes.rb +184 -0
  135. data/lib/milk_tea/core/semantic_analyzer/calls.rb +992 -0
  136. data/lib/milk_tea/core/semantic_analyzer/expressions.rb +1735 -0
  137. data/lib/milk_tea/core/semantic_analyzer/flow_refinement.rb +355 -0
  138. data/lib/milk_tea/core/semantic_analyzer/foreign_functions.rb +155 -0
  139. data/lib/milk_tea/core/semantic_analyzer/function_binding.rb +354 -0
  140. data/lib/milk_tea/core/semantic_analyzer/generics.rb +383 -0
  141. data/lib/milk_tea/core/semantic_analyzer/interface_conformance.rb +78 -0
  142. data/lib/milk_tea/core/semantic_analyzer/module_context.rb +35 -0
  143. data/lib/milk_tea/core/semantic_analyzer/name_resolution.rb +1438 -0
  144. data/lib/milk_tea/core/semantic_analyzer/nullability.rb +421 -0
  145. data/lib/milk_tea/core/semantic_analyzer/statements.rb +1308 -0
  146. data/lib/milk_tea/core/semantic_analyzer/top_level.rb +588 -0
  147. data/lib/milk_tea/core/semantic_analyzer/type_compatibility.rb +307 -0
  148. data/lib/milk_tea/core/semantic_analyzer/type_declaration.rb +851 -0
  149. data/lib/milk_tea/core/semantic_analyzer.rb +327 -0
  150. data/lib/milk_tea/core/token.rb +26 -0
  151. data/lib/milk_tea/core/token_stream.rb +30 -0
  152. data/lib/milk_tea/core/types/layout.rb +243 -0
  153. data/lib/milk_tea/core/types/predicates.rb +609 -0
  154. data/lib/milk_tea/core/types/registry.rb +83 -0
  155. data/lib/milk_tea/core/types/types.rb +1696 -0
  156. data/lib/milk_tea/core/types/visitor.rb +442 -0
  157. data/lib/milk_tea/core.rb +25 -0
  158. data/lib/milk_tea/dap/backends/lldb_dap.rb +158 -0
  159. data/lib/milk_tea/dap/protocol.rb +58 -0
  160. data/lib/milk_tea/dap/server/breakpoints.rb +66 -0
  161. data/lib/milk_tea/dap/server/debug_map.rb +291 -0
  162. data/lib/milk_tea/dap/server/handlers.rb +374 -0
  163. data/lib/milk_tea/dap/server/launch.rb +261 -0
  164. data/lib/milk_tea/dap/server/lldb_backend.rb +380 -0
  165. data/lib/milk_tea/dap/server/pause_diagnostics.rb +109 -0
  166. data/lib/milk_tea/dap/server/utilities.rb +160 -0
  167. data/lib/milk_tea/dap/server/wire.rb +55 -0
  168. data/lib/milk_tea/dap/server.rb +131 -0
  169. data/lib/milk_tea/dap/session.rb +152 -0
  170. data/lib/milk_tea/dap.rb +6 -0
  171. data/lib/milk_tea/lsp/dependency_resolution.rb +52 -0
  172. data/lib/milk_tea/lsp/diagnostics.rb +611 -0
  173. data/lib/milk_tea/lsp/protocol.rb +104 -0
  174. data/lib/milk_tea/lsp/server/call_hierarchy.rb +274 -0
  175. data/lib/milk_tea/lsp/server/code_actions.rb +446 -0
  176. data/lib/milk_tea/lsp/server/code_lens.rb +97 -0
  177. data/lib/milk_tea/lsp/server/completion.rb +1099 -0
  178. data/lib/milk_tea/lsp/server/configuration.rb +167 -0
  179. data/lib/milk_tea/lsp/server/debug_info.rb +45 -0
  180. data/lib/milk_tea/lsp/server/definition.rb +779 -0
  181. data/lib/milk_tea/lsp/server/diagnostics_scheduling.rb +239 -0
  182. data/lib/milk_tea/lsp/server/execute_command.rb +25 -0
  183. data/lib/milk_tea/lsp/server/folding_range.rb +153 -0
  184. data/lib/milk_tea/lsp/server/formatting.rb +575 -0
  185. data/lib/milk_tea/lsp/server/hover.rb +1465 -0
  186. data/lib/milk_tea/lsp/server/inlay_hints.rb +204 -0
  187. data/lib/milk_tea/lsp/server/lifecycle.rb +234 -0
  188. data/lib/milk_tea/lsp/server/linked_editing_range.rb +43 -0
  189. data/lib/milk_tea/lsp/server/on_type_formatting.rb +73 -0
  190. data/lib/milk_tea/lsp/server/progress.rb +47 -0
  191. data/lib/milk_tea/lsp/server/references.rb +433 -0
  192. data/lib/milk_tea/lsp/server/rename.rb +598 -0
  193. data/lib/milk_tea/lsp/server/selection_range.rb +130 -0
  194. data/lib/milk_tea/lsp/server/semantic_tokens.rb +1745 -0
  195. data/lib/milk_tea/lsp/server/signature_help.rb +200 -0
  196. data/lib/milk_tea/lsp/server/text_documents.rb +125 -0
  197. data/lib/milk_tea/lsp/server/type_hierarchy.rb +167 -0
  198. data/lib/milk_tea/lsp/server/utilities.rb +520 -0
  199. data/lib/milk_tea/lsp/server.rb +415 -0
  200. data/lib/milk_tea/lsp/workspace/analysis.rb +209 -0
  201. data/lib/milk_tea/lsp/workspace/caches.rb +260 -0
  202. data/lib/milk_tea/lsp/workspace/collection.rb +98 -0
  203. data/lib/milk_tea/lsp/workspace/definition_index.rb +184 -0
  204. data/lib/milk_tea/lsp/workspace/dependency_graph.rb +282 -0
  205. data/lib/milk_tea/lsp/workspace/store.rb +154 -0
  206. data/lib/milk_tea/lsp/workspace/utilities.rb +490 -0
  207. data/lib/milk_tea/lsp/workspace.rb +127 -0
  208. data/lib/milk_tea/lsp.rb +13 -0
  209. data/lib/milk_tea/packages/atomic_write.rb +63 -0
  210. data/lib/milk_tea/packages/dependency_solver.rb +194 -0
  211. data/lib/milk_tea/packages/graph.rb +139 -0
  212. data/lib/milk_tea/packages/lock.rb +421 -0
  213. data/lib/milk_tea/packages/manager_cli.rb +485 -0
  214. data/lib/milk_tea/packages/manifest.rb +356 -0
  215. data/lib/milk_tea/packages/manifest_editor.rb +134 -0
  216. data/lib/milk_tea/packages/registry_metadata_provider.rb +34 -0
  217. data/lib/milk_tea/packages/registry_store.rb +420 -0
  218. data/lib/milk_tea/packages/services.rb +41 -0
  219. data/lib/milk_tea/packages/source_cache.rb +71 -0
  220. data/lib/milk_tea/packages/source_fetcher.rb +124 -0
  221. data/lib/milk_tea/packages/source_resolver.rb +389 -0
  222. data/lib/milk_tea/packages/version.rb +164 -0
  223. data/lib/milk_tea/packages.rb +16 -0
  224. data/lib/milk_tea/tooling/asset_pack.rb +141 -0
  225. data/lib/milk_tea/tooling/build.rb +1124 -0
  226. data/lib/milk_tea/tooling/build_cache.rb +240 -0
  227. data/lib/milk_tea/tooling/cli.rb +2688 -0
  228. data/lib/milk_tea/tooling/cst_formatter.rb +13 -0
  229. data/lib/milk_tea/tooling/debug_info_formatter.rb +456 -0
  230. data/lib/milk_tea/tooling/debug_map.rb +248 -0
  231. data/lib/milk_tea/tooling/docs_app.rb +369 -0
  232. data/lib/milk_tea/tooling/error_formatter.rb +86 -0
  233. data/lib/milk_tea/tooling/formatter.rb +787 -0
  234. data/lib/milk_tea/tooling/linter/doc_tags.rb +212 -0
  235. data/lib/milk_tea/tooling/linter/fix_engine.rb +237 -0
  236. data/lib/milk_tea/tooling/linter/flow_rules.rb +380 -0
  237. data/lib/milk_tea/tooling/linter/imports_platform.rb +841 -0
  238. data/lib/milk_tea/tooling/linter/release_rules.rb +744 -0
  239. data/lib/milk_tea/tooling/linter/reserved_names.rb +199 -0
  240. data/lib/milk_tea/tooling/linter/rules.rb +593 -0
  241. data/lib/milk_tea/tooling/linter/source_helpers.rb +407 -0
  242. data/lib/milk_tea/tooling/linter/trailing_comma.rb +139 -0
  243. data/lib/milk_tea/tooling/linter/visitors.rb +1286 -0
  244. data/lib/milk_tea/tooling/linter.rb +798 -0
  245. data/lib/milk_tea/tooling/project_scaffold.rb +82 -0
  246. data/lib/milk_tea/tooling/public/css/docs.css +166 -0
  247. data/lib/milk_tea/tooling/public/js/docs.js +94 -0
  248. data/lib/milk_tea/tooling/run.rb +408 -0
  249. data/lib/milk_tea/tooling/templates/wasm_shell.html +48 -0
  250. data/lib/milk_tea/tooling/toolchain_cli.rb +157 -0
  251. data/lib/milk_tea/tooling/views/404.erb +7 -0
  252. data/lib/milk_tea/tooling/views/index.erb +87 -0
  253. data/lib/milk_tea/tooling/views/layout.erb +69 -0
  254. data/lib/milk_tea/tooling/views/module.erb +97 -0
  255. data/lib/milk_tea/tooling/views/stdlib.erb +21 -0
  256. data/lib/milk_tea/tooling.rb +20 -0
  257. data/lib/milk_tea.rb +8 -0
  258. metadata +426 -0
@@ -0,0 +1,430 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "lexer/character_classes"
4
+ require_relative "lexer/trivia"
5
+ require_relative "lexer/indentation"
6
+ require_relative "lexer/numbers"
7
+ require_relative "lexer/strings"
8
+ require_relative "lexer/heredocs"
9
+ require_relative "lexer/format_strings"
10
+ require_relative "lexer/symbols"
11
+ require_relative "lexer/recovery"
12
+
13
+ module MilkTea
14
+ class LexError < StandardError
15
+ attr_reader :line, :column, :path
16
+
17
+ def initialize(message, line:, column:, path: nil)
18
+ @line = line
19
+ @column = column
20
+ @path = path
21
+
22
+ location = [path, "#{line}:#{column}"].compact.join(":")
23
+ super(location.empty? ? message : "#{message} at #{location}")
24
+ end
25
+
26
+ def code
27
+ "lex/error"
28
+ end
29
+ end
30
+
31
+ class Lexer
32
+ LexResult = Data.define(:tokens, :trivia)
33
+ StringLexResult = Data.define(:consumed_lines, :next_index)
34
+ StringSegment = Data.define(:next_index, :value, :recovered) do
35
+ def initialize(next_index:, value:, recovered: false)
36
+ super(next_index:, value:, recovered:)
37
+ end
38
+ end
39
+
40
+ TOP_LEVEL_RESYNC_PREFIXES = %w[
41
+ attribute const enum external flags foreign function include import interface
42
+ link opaque public static_assert struct type union var variant extending event
43
+ ].freeze
44
+
45
+ LINE_CONTINUATION_OPERATORS = %i[
46
+ dot_dot
47
+ plus minus star slash percent
48
+ pipe amp caret
49
+ or and
50
+ equal_equal bang_equal
51
+ less less_equal greater greater_equal
52
+ shift_left shift_right
53
+ ].freeze
54
+
55
+ THREE_CHAR_TOKENS = {
56
+ "..." => :ellipsis,
57
+ "<<=" => :shift_left_equal,
58
+ ">>=" => :shift_right_equal,
59
+ }.freeze
60
+
61
+ TWO_CHAR_TOKENS = {
62
+ "->" => :arrow,
63
+ ".." => :dot_dot,
64
+ "<<" => :shift_left,
65
+ ">>" => :shift_right,
66
+ "+=" => :plus_equal,
67
+ "-=" => :minus_equal,
68
+ "*=" => :star_equal,
69
+ "/=" => :slash_equal,
70
+ "%=" => :percent_equal,
71
+ "&=" => :amp_equal,
72
+ "|=" => :pipe_equal,
73
+ "^=" => :caret_equal,
74
+ "==" => :equal_equal,
75
+ "!=" => :bang_equal,
76
+ "<=" => :less_equal,
77
+ ">=" => :greater_equal,
78
+ }.freeze
79
+
80
+ ONE_CHAR_TOKENS = {
81
+ "&" => :amp,
82
+ "@" => :at,
83
+ ":" => :colon,
84
+ "," => :comma,
85
+ "^" => :caret,
86
+ "." => :dot,
87
+ "(" => :lparen,
88
+ ")" => :rparen,
89
+ "|" => :pipe,
90
+ "[" => :lbracket,
91
+ "]" => :rbracket,
92
+ "?" => :question,
93
+ "=" => :equal,
94
+ "+" => :plus,
95
+ "-" => :minus,
96
+ "*" => :star,
97
+ "/" => :slash,
98
+ "%" => :percent,
99
+ "<" => :less,
100
+ ">" => :greater,
101
+ "~" => :tilde,
102
+ }.freeze
103
+
104
+ INTEGER_SUFFIX_STRINGS = %w[ub us ul iz b s i u l z].sort_by { |s| -s.length }.freeze
105
+
106
+ # Byte-classification lookup tables (indexed 0..255). Scanning by raw byte
107
+ # via String#getbyte avoids allocating a one-character String and running
108
+ # the regex engine for every source character, which dominated lexing.
109
+ IDENT_START_BYTE = Array.new(256) { |b| b.chr.match?(/[A-Za-z_]/) }.freeze
110
+ IDENT_PART_BYTE = Array.new(256) { |b| b.chr.match?(/[A-Za-z0-9_]/) }.freeze
111
+ DIGIT_BYTE = Array.new(256) { |b| b.chr.match?(/[0-9]/) }.freeze
112
+ NUMERIC_PART_BYTE = Array.new(256) { |b| b.chr.match?(/[0-9_]/) }.freeze
113
+ HEX_DIGIT_BYTE = Array.new(256) { |b| b.chr.match?(/[0-9a-fA-F_]/) }.freeze
114
+ BIN_DIGIT_BYTE = Array.new(256) { |b| b.chr.match?(/[01_]/) }.freeze
115
+ SPACE_BYTE = " ".ord
116
+
117
+ def self.lex(source, path: nil, mode: :syntax_only, recovery_errors: nil)
118
+ result = new(source, path: path, mode:, recovery_errors:).lex
119
+ mode == :with_trivia ? result.tokens : result
120
+ end
121
+
122
+ def self.lex_with_trivia(source, path: nil)
123
+ new(source, path: path, mode: :with_trivia).lex
124
+ end
125
+
126
+ def initialize(source, path: nil, mode: :syntax_only, recovery_errors: nil)
127
+ @source = source.gsub(/\r\n?/, "\n")
128
+ @path = path
129
+ @mode = mode
130
+ @recovery_errors = recovery_errors
131
+ @tokens = []
132
+ @trivia = []
133
+ @pending_leading_trivia = []
134
+ @indent_stack = [0]
135
+ @grouping_depth = 0
136
+ @grouping_start_line = 0
137
+ @grouping_start_column = 0
138
+ @line_count = @source.empty? ? 1 : @source.lines.count
139
+ @continuation_pending = false
140
+ end
141
+
142
+ def lex
143
+ lines = @source.each_line.to_a
144
+ line_offset = 0
145
+ line_number = 1
146
+ line_index = 0
147
+
148
+ while line_index < lines.length
149
+ raw_line = lines[line_index]
150
+ has_newline = raw_line.end_with?("\n")
151
+ line = raw_line.delete_suffix("\n").b
152
+ consumed_lines = lex_line(lines, line_index, line, line_number, line_offset, has_newline:)
153
+
154
+ consumed_lines.times do |delta|
155
+ line_offset += lines.fetch(line_index + delta).bytesize
156
+ end
157
+ line_number += consumed_lines
158
+ line_index += consumed_lines
159
+ end
160
+
161
+ if @grouping_depth.positive?
162
+ if @recovery_errors
163
+ @recovery_errors << LexError.new("unclosed grouping delimiter", line: @grouping_start_line, column: @grouping_start_column, path: @path)
164
+ @grouping_depth = 0
165
+ else
166
+ raise LexError.new("unclosed grouping delimiter", line: @line_count, column: 1, path: @path)
167
+ end
168
+ end
169
+
170
+ while @indent_stack.length > 1
171
+ @indent_stack.pop
172
+ @tokens << token(:dedent, "", nil, @line_count, 1, start_offset: @source.bytesize, end_offset: @source.bytesize)
173
+ end
174
+
175
+ @tokens << token(:eof, "", nil, @line_count + 1, 1, start_offset: @source.bytesize, end_offset: @source.bytesize)
176
+ return LexResult.new(tokens: @tokens, trivia: @trivia) if with_trivia?
177
+
178
+ @tokens
179
+ end
180
+
181
+ private
182
+
183
+ include CharacterClasses
184
+ include Trivia
185
+ include Indentation
186
+ include Numbers
187
+ include Strings
188
+ include Heredocs
189
+ include FormatStrings
190
+ include Symbols
191
+ include Recovery
192
+
193
+ def lex_line(lines, line_index, line, line_number, line_offset, has_newline:)
194
+ tab_index = line.index("\t")
195
+ if tab_index
196
+ error = LexError.new("tabs are not allowed; use 4 spaces for indentation", line: line_number, column: tab_index + 1, path: @path)
197
+ if @recovery_errors
198
+ @recovery_errors << error
199
+ line = line.gsub("\t", " ")
200
+ else
201
+ raise error
202
+ end
203
+ end
204
+
205
+ if line.strip.empty?
206
+ register_detached_line_trivia(:blank_line, line, line_number, line_offset, has_newline:)
207
+ return 1
208
+ end
209
+
210
+ if line.lstrip.start_with?("#")
211
+ register_detached_line_trivia(:comment, line, line_number, line_offset, has_newline:)
212
+ return 1
213
+ end
214
+
215
+ index = leading_space_count(line)
216
+ if @recovery_errors && @grouping_depth.positive? && index.zero? && top_level_resync_line?(line)
217
+ @recovery_errors << LexError.new("unclosed grouping delimiter", line: @grouping_start_line, column: @grouping_start_column, path: @path)
218
+ @grouping_depth = 0
219
+ @tokens << token(:newline, "\n", nil, line_number, 1, start_offset: line_offset, end_offset: line_offset)
220
+ end
221
+
222
+ if with_trivia? && index.positive?
223
+ push_pending_leading_trivia(
224
+ TriviaToken.new(
225
+ kind: :space,
226
+ text: line[0...index],
227
+ line: line_number,
228
+ column: 1,
229
+ start_offset: line_offset,
230
+ end_offset: line_offset + index,
231
+ ),
232
+ )
233
+ end
234
+
235
+ if @grouping_depth.zero?
236
+ if @continuation_pending
237
+ elsif @recovery_errors
238
+ begin
239
+ emit_indentation(index, line_number, line_offset)
240
+ rescue LexError => e
241
+ @recovery_errors << e
242
+ recover_indentation(index, line_number, line_offset)
243
+ end
244
+ else
245
+ emit_indentation(index, line_number, line_offset)
246
+ end
247
+ end
248
+ @continuation_pending = false
249
+
250
+ while index < line.length
251
+ char = line[index]
252
+
253
+ if char == " "
254
+ if with_trivia?
255
+ span_start = index
256
+ index += 1 while index < line.length && line[index] == " "
257
+ push_pending_leading_trivia(
258
+ TriviaToken.new(
259
+ kind: :space,
260
+ text: line[span_start...index],
261
+ line: line_number,
262
+ column: span_start + 1,
263
+ start_offset: line_offset + span_start,
264
+ end_offset: line_offset + index,
265
+ ),
266
+ )
267
+ next
268
+ end
269
+
270
+ index += 1
271
+ next
272
+ end
273
+
274
+ if char == "#"
275
+ if with_trivia?
276
+ comment_end = line.length
277
+ comment_text = line[index...comment_end]
278
+ append_trailing_or_pending(
279
+ TriviaToken.new(
280
+ kind: :comment,
281
+ text: comment_text,
282
+ line: line_number,
283
+ column: index + 1,
284
+ start_offset: line_offset + index,
285
+ end_offset: line_offset + comment_end,
286
+ ),
287
+ )
288
+ end
289
+ break
290
+ end
291
+
292
+ if char == "c" && heredoc_start?(line, index, cstring: true)
293
+ return lex_heredoc(lines, line_index, index, line_number, line_offset, cstring: true)
294
+ end
295
+
296
+ if char == "c" && line[index + 1] == "<" && line[index + 2] == "-" && identifier_start?(line[index + 3])
297
+ error = LexError.new("expected '<<-' for heredoc string; did you mean 'c<<-#{identifier_start_token(line, index + 3)}'?", line: line_number, column: index + 1, path: @path)
298
+ if @recovery_errors
299
+ @recovery_errors << error
300
+ else
301
+ raise error
302
+ end
303
+ end
304
+ if char == "c" && line[index + 1] == '"'
305
+ result = lex_string(lines, line_index, line, index, line_number, line_offset:, cstring: true)
306
+ return result.consumed_lines if result.consumed_lines > 1
307
+
308
+ index = result.next_index
309
+ next
310
+ end
311
+
312
+ if char == "f" && heredoc_start?(line, index, format: true)
313
+ return lex_heredoc(lines, line_index, index, line_number, line_offset, cstring: false, format: true)
314
+ end
315
+
316
+ if char == "f" && line[index + 1] == "<" && line[index + 2] == "-" && identifier_start?(line[index + 3])
317
+ error = LexError.new("expected '<<-' for heredoc string; did you mean 'f<<-#{identifier_start_token(line, index + 3)}'?", line: line_number, column: index + 1, path: @path)
318
+ if @recovery_errors
319
+ @recovery_errors << error
320
+ else
321
+ raise error
322
+ end
323
+ end
324
+
325
+ if char == "f" && line[index + 1] == '"'
326
+ index = lex_format_string(line, index, line_number, line_offset:)
327
+ next
328
+ end
329
+
330
+ if char == "<" && heredoc_start?(line, index)
331
+ return lex_heredoc(lines, line_index, index, line_number, line_offset, cstring: false)
332
+ end
333
+
334
+ if char == '"'
335
+ result = lex_string(lines, line_index, line, index, line_number, line_offset:)
336
+ return result.consumed_lines if result.consumed_lines > 1
337
+
338
+ index = result.next_index
339
+ next
340
+ end
341
+
342
+ if char == "'"
343
+ index = lex_char_literal(line, index, line_number, line_offset:)
344
+ next
345
+ end
346
+
347
+ if identifier_start?(char)
348
+ index = lex_identifier(line, index, line_number, line_offset:)
349
+ next
350
+ end
351
+
352
+ if digit?(char)
353
+ index = lex_number(line, index, line_number, line_offset:)
354
+ next
355
+ end
356
+
357
+ index = lex_symbol(line, index, line_number, line_offset:)
358
+ end
359
+
360
+ newline_start = line_offset + line.length
361
+ newline_end = has_newline ? (newline_start + 1) : newline_start
362
+ if @grouping_depth.zero?
363
+ if LINE_CONTINUATION_OPERATORS.include?(@tokens.last&.type)
364
+ @continuation_pending = true
365
+ else
366
+ @tokens << token(:newline, "\n", nil, line_number, line.length + 1, start_offset: newline_start, end_offset: newline_end)
367
+ end
368
+ elsif with_trivia? && has_newline
369
+ append_trailing_or_pending(
370
+ TriviaToken.new(
371
+ kind: :newline,
372
+ text: "\n",
373
+ line: line_number,
374
+ column: line.length + 1,
375
+ start_offset: newline_start,
376
+ end_offset: newline_end,
377
+ ),
378
+ )
379
+ end
380
+
381
+ 1
382
+ end
383
+
384
+ # ── newline emission ────────────────────────────────────────────
385
+
386
+ def emit_line_newline(line, line_number, line_offset, has_newline)
387
+ newline_start = line_offset + line.bytesize
388
+ newline_end = has_newline ? (newline_start + 1) : newline_start
389
+
390
+ if @grouping_depth.zero?
391
+ @tokens << token(:newline, "\n", nil, line_number, line.bytesize + 1, start_offset: newline_start, end_offset: newline_end)
392
+ elsif with_trivia? && has_newline
393
+ append_trailing_or_pending(
394
+ TriviaToken.new(
395
+ kind: :newline,
396
+ text: "\n",
397
+ line: line_number,
398
+ column: line.bytesize + 1,
399
+ start_offset: newline_start,
400
+ end_offset: newline_end,
401
+ ),
402
+ )
403
+ end
404
+ end
405
+
406
+ # ── token construction ──────────────────────────────────────────
407
+
408
+ def token(type, lexeme, literal, line, column, start_offset:, end_offset:)
409
+ Token.new(
410
+ type:,
411
+ lexeme:,
412
+ literal:,
413
+ line:,
414
+ column:,
415
+ start_offset:,
416
+ end_offset:,
417
+ leading_trivia: consume_pending_leading_trivia,
418
+ trailing_trivia: [],
419
+ )
420
+ end
421
+
422
+ def consume_pending_leading_trivia
423
+ return [] unless with_trivia?
424
+
425
+ trivia = @pending_leading_trivia
426
+ @pending_leading_trivia = []
427
+ trivia
428
+ end
429
+ end
430
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module MilkTea
4
+ class Lowerer
5
+ class Artifacts
6
+ attr_accessor :synthetic_structs, :synthetic_enums, :synthetic_functions, :synthetic_constants
7
+ attr_accessor :emitted_declarations, :external_layout_assertions, :emitted_external_layout_pairs
8
+ attr_accessor :lowered_function_linkage_names, :event_runtime_infos
9
+ attr_accessor :subscription_runtime_emitted, :event_error_enum_emitted
10
+
11
+ def initialize
12
+ @synthetic_structs = []
13
+ @synthetic_enums = []
14
+ @synthetic_functions = []
15
+ @synthetic_constants = []
16
+ @emitted_declarations = []
17
+ @external_layout_assertions = []
18
+ @emitted_external_layout_pairs = {}
19
+ @lowered_function_linkage_names = {}
20
+ @event_runtime_infos = {}
21
+ @subscription_runtime_emitted = false
22
+ @event_error_enum_emitted = false
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,245 @@
1
+ # frozen_string_literal: true
2
+
3
+ module MilkTea
4
+ module LowererAsync
5
+ private
6
+
7
+ def analyze_async_function(binding, statements)
8
+ env = empty_env
9
+ void_ptr = pointer_to(@ctx.types.fetch("void"))
10
+ wake_type = Types::Registry.function(
11
+ nil,
12
+ params: [Types::Registry.parameter("frame", void_ptr)],
13
+ return_type: @ctx.types.fetch("void"),
14
+ )
15
+ param_fields = {}
16
+ local_fields = {}
17
+ await_fields = {}
18
+ await_counter = 0
19
+
20
+ binding.body_params.each do |param_binding|
21
+ pointer = binding.type.receiver_type && binding.type.receiver_editable && param_binding.name == "this"
22
+ field_type = pointer ? pointer_to(param_binding.type) : param_binding.type
23
+ field_name = "param_#{param_binding.name}"
24
+ param_fields[param_binding.name] = {
25
+ field_name:,
26
+ type: field_type,
27
+ param_type: param_binding.type,
28
+ mutable: param_binding.mutable,
29
+ pointer:,
30
+ }
31
+ env[:scopes].last[param_binding.name] = local_binding(
32
+ type: param_binding.type,
33
+ linkage_name: field_name,
34
+ mutable: param_binding.mutable,
35
+ pointer:,
36
+ )
37
+ end
38
+ env[:return_context] = {
39
+ return_type: binding.body_return_type,
40
+ active_defers: [],
41
+ local_defers: [],
42
+ allow_return: true,
43
+ }
44
+
45
+ await_counter = analyze_async_statements!(statements, await_counter, env, param_fields, local_fields, await_fields)
46
+
47
+ {
48
+ task_type: binding.type.return_type,
49
+ result_type: binding.body_return_type,
50
+ void_ptr:,
51
+ wake_type:,
52
+ param_fields:,
53
+ local_fields:,
54
+ await_fields:,
55
+ format_str_fields: {},
56
+ }
57
+ end
58
+
59
+ # Recursively scan nested statement bodies for await slots, assigning state IDs.
60
+ # Returns the updated await_counter.
61
+ def analyze_async_statements!(statements, await_counter, env, param_fields, local_fields, await_fields)
62
+ statements.each do |statement|
63
+ case statement
64
+ when AST::LocalDecl
65
+ type, storage_type = async_local_decl_types(statement, env:)
66
+ local_field_key = async_local_decl_field_key(statement)
67
+ local_fields[local_field_key] ||= { field_name: async_local_decl_field_name(statement), type:, storage_type:, mutable: statement.kind == :var }
68
+ if statement.value.is_a?(AST::AwaitExpr)
69
+ await_fields[statement.value.object_id] = build_async_await_field_info(statement.value, await_counter, env:, param_fields:, local_fields:)
70
+ await_counter += 1
71
+ end
72
+ if bind_let_else_local?(statement)
73
+ env[:scopes].last[statement.name] = local_binding(
74
+ type:,
75
+ storage_type:,
76
+ linkage_name: statement.name,
77
+ mutable: statement.kind == :var,
78
+ pointer: false,
79
+ const_value: statement.else_body ? nil : statement.kind == :let && statement.value ? compile_time_const_value(statement.value, env:) : nil,
80
+ )
81
+ end
82
+ await_counter = analyze_async_statements!(statement.else_body, await_counter, env, param_fields, local_fields, await_fields) if statement.else_body
83
+ when AST::Assignment
84
+ if statement.value.is_a?(AST::AwaitExpr)
85
+ await_fields[statement.value.object_id] = build_async_await_field_info(statement.value, await_counter, env:, param_fields:, local_fields:)
86
+ await_counter += 1
87
+ end
88
+ when AST::ExpressionStmt
89
+ if statement.expression.is_a?(AST::AwaitExpr)
90
+ await_fields[statement.expression.object_id] = build_async_await_field_info(statement.expression, await_counter, env:, param_fields:, local_fields:)
91
+ await_counter += 1
92
+ end
93
+ when AST::ReturnStmt
94
+ if statement.value&.is_a?(AST::AwaitExpr)
95
+ await_fields[statement.value.object_id] = build_async_await_field_info(statement.value, await_counter, env:, param_fields:, local_fields:)
96
+ await_counter += 1
97
+ end
98
+ when AST::IfStmt
99
+ statement.branches.each do |branch|
100
+ await_counter = analyze_async_statements!(branch.body, await_counter, env, param_fields, local_fields, await_fields)
101
+ end
102
+ await_counter = analyze_async_statements!(statement.else_body, await_counter, env, param_fields, local_fields, await_fields) if statement.else_body
103
+ when AST::WhileStmt
104
+ await_counter = analyze_async_statements!(statement.body, await_counter, env, param_fields, local_fields, await_fields)
105
+ when AST::ForStmt
106
+ if range_iterable?(statement.iterable)
107
+ loop_type = infer_range_loop_type(statement.iterable, env:)
108
+ local_fields[statement.name] ||= { field_name: "local_#{statement.name}", type: loop_type, storage_type: loop_type, mutable: true }
109
+ stop_field_name = "local_#{statement.name}_stop"
110
+ local_fields[stop_field_name] ||= { field_name: stop_field_name, type: loop_type, storage_type: loop_type, mutable: true }
111
+ else
112
+ statement.bindings.each_with_index do |binding, index|
113
+ iterable_type = infer_expression_type(statement.iterables[index], env:)
114
+ element_type = collection_loop_type(iterable_type)
115
+ binding_type = collection_loop_binding_type(iterable_type, element_type) || element_type
116
+ local_fields[binding.name] ||= { field_name: "local_#{binding.name}", type: binding_type, storage_type: binding_type, mutable: true }
117
+ iterable_field_name = async_collection_iterable_field_name(statement, index)
118
+ iterable_field_key = async_collection_iterable_field_key(statement, index)
119
+ local_fields[iterable_field_key] ||= { field_name: iterable_field_name, type: iterable_type, storage_type: iterable_type, mutable: true }
120
+ end
121
+ index_field_name = async_collection_index_field_name(statement)
122
+ index_field_key = async_collection_index_field_key(statement)
123
+ local_fields[index_field_key] ||= { field_name: index_field_name, type: @ctx.types.fetch("ptr_uint"), storage_type: @ctx.types.fetch("ptr_uint"), mutable: true }
124
+ end
125
+ await_counter = analyze_async_statements!(statement.body, await_counter, env, param_fields, local_fields, await_fields)
126
+ when AST::MatchStmt
127
+ scrutinee_type = infer_expression_type(statement.expression, env:)
128
+ statement.arms.each do |arm|
129
+ arm_env = duplicate_env(env)
130
+ bind_async_variant_match_arm_env!(arm_env, scrutinee_type, arm)
131
+ arm_await_count = await_counter
132
+ await_counter = analyze_async_statements!(arm.body, await_counter, arm_env, param_fields, local_fields, await_fields)
133
+ if arm.binding_name && await_counter > arm_await_count
134
+ field_key = async_match_binding_field_key(arm)
135
+ unless local_fields.key?(field_key)
136
+ arm_name = variant_match_arm_name_from_pattern(arm.pattern)
137
+ if arm_name
138
+ arm_binding = arm_env[:scopes].last[arm.binding_name]
139
+ payload_type = arm_binding && arm_binding[:type]
140
+ if payload_type
141
+ local_fields[field_key] = { field_name: async_match_binding_field_name(arm), type: payload_type, storage_type: payload_type, mutable: false }
142
+ end
143
+ end
144
+ end
145
+ end
146
+ end
147
+ when AST::UnsafeStmt
148
+ await_counter = analyze_async_statements!(statement.body, await_counter, env, param_fields, local_fields, await_fields)
149
+ when AST::DeferStmt
150
+ if statement.body
151
+ cleanup_env = duplicate_env(env)
152
+ cleanup_env[:return_context] = cleanup_env[:return_context]&.merge(allow_return: false)
153
+ await_counter = analyze_async_statements!(statement.body, await_counter, cleanup_env, param_fields, local_fields, await_fields)
154
+ end
155
+ if statement.expression.is_a?(AST::AwaitExpr)
156
+ await_fields[statement.expression.object_id] = build_async_await_field_info(statement.expression, await_counter, env:, param_fields:, local_fields:)
157
+ await_counter += 1
158
+ end
159
+ else
160
+ nil
161
+ end
162
+ end
163
+ await_counter
164
+ end
165
+
166
+ def async_local_decl_types(statement, env:)
167
+ storage_type = if statement.else_body
168
+ infer_expression_type(statement.value, env:)
169
+ elsif statement.type
170
+ resolve_type_ref(statement.type)
171
+ else
172
+ infer_expression_type(statement.value, env:)
173
+ end
174
+ type = if statement.else_body
175
+ statement.type ? resolve_type_ref(statement.type) : let_else_success_type(storage_type)
176
+ else
177
+ storage_type
178
+ end
179
+
180
+ [type, storage_type]
181
+ end
182
+
183
+ def async_collection_iterable_field_key(statement, index = 0)
184
+ "__async_for_iterable_#{statement.line}_#{index}"
185
+ end
186
+
187
+ def async_collection_iterable_field_name(statement, index = 0)
188
+ "for_iterable_#{statement.line}_#{index}"
189
+ end
190
+
191
+ def async_collection_index_field_key(statement)
192
+ "__async_for_index_#{statement.line}"
193
+ end
194
+
195
+ def async_collection_index_field_name(statement)
196
+ "for_index_#{statement.line}"
197
+ end
198
+
199
+ def build_async_await_field_info(await_expression, await_counter, env:, param_fields:, local_fields:)
200
+ task_expression = await_expression.expression
201
+ reused_field_name = reusable_async_await_task_field_name(task_expression, param_fields:, local_fields:)
202
+ {
203
+ field_name: reused_field_name || "await_#{await_counter}",
204
+ task_type: infer_expression_type(task_expression, env:),
205
+ result_type: infer_expression_type(await_expression, env:),
206
+ state: await_counter + 1,
207
+ reuse_existing_storage: !reused_field_name.nil?,
208
+ }
209
+ end
210
+
211
+ def reusable_async_await_task_field_name(task_expression, param_fields:, local_fields:)
212
+ return unless task_expression.is_a?(AST::Identifier)
213
+ return local_fields.fetch(task_expression.name)[:field_name] if local_fields.key?(task_expression.name)
214
+ return param_fields.fetch(task_expression.name)[:field_name] if param_fields.key?(task_expression.name)
215
+
216
+ nil
217
+ end
218
+
219
+ def build_async_frame_type(frame_linkage_name, async_info)
220
+ fields = {
221
+ "ready" => @ctx.types.fetch("bool"),
222
+ "cancelled" => @ctx.types.fetch("bool"),
223
+ "waiter_frame" => async_info[:void_ptr],
224
+ "waiter" => async_info[:wake_type],
225
+ }
226
+ fields["state"] = @ctx.types.fetch("int") unless async_info[:await_fields].empty?
227
+ unless async_info[:result_type] == @ctx.types.fetch("void")
228
+ fields["result"] = async_info[:result_type]
229
+ end
230
+ async_info[:param_fields].each_value do |field_info|
231
+ fields[field_info[:field_name]] = field_info[:type]
232
+ end
233
+ async_info[:local_fields].each_value do |field_info|
234
+ fields[field_info[:field_name]] = field_info[:storage_type]
235
+ end
236
+ async_info[:await_fields].each_value do |field_info|
237
+ next if fields.key?(field_info[:field_name])
238
+
239
+ fields[field_info[:field_name]] = field_info[:task_type]
240
+ end
241
+
242
+ Types::Struct.new(frame_linkage_name).define_fields(fields)
243
+ end
244
+ end
245
+ end