aura-lang 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +154 -0
- data/LICENSE +21 -0
- data/README.md +120 -0
- data/Rakefile +59 -0
- data/aura-lang.gemspec +29 -0
- data/bin/aura +151 -0
- data/examples/assistant.aura +12 -0
- data/examples/chatbot.aura +7 -0
- data/examples/hello.aura +10 -0
- data/examples/mnist_classifier.aura +36 -0
- data/examples/sentiment.aura +13 -0
- data/examples/transfer_api.aura +25 -0
- data/lib/aura/analyzer.rb +125 -0
- data/lib/aura/codegen.rb +636 -0
- data/lib/aura/diagnostics.rb +86 -0
- data/lib/aura/docker.rb +81 -0
- data/lib/aura/emitter.rb +61 -0
- data/lib/aura/parser.rb +208 -0
- data/lib/aura/transformer.rb +152 -0
- data/lib/aura/vercel.rb +88 -0
- data/lib/aura/version.rb +8 -0
- data/lib/aura.rb +125 -0
- metadata +158 -0
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Aura
|
|
4
|
+
# Raised when source text cannot be parsed. Carries the line/column of the
|
|
5
|
+
# failure plus a rendered snippet with a caret, so users get a real
|
|
6
|
+
# diagnostic instead of a raw Parslet backtrace.
|
|
7
|
+
class ParseError < StandardError
|
|
8
|
+
attr_reader :line, :column
|
|
9
|
+
|
|
10
|
+
def initialize(message, line: nil, column: nil)
|
|
11
|
+
@line = line
|
|
12
|
+
@column = column
|
|
13
|
+
super(message)
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Raised by the semantic analysis pass for problems a grammar cannot catch,
|
|
18
|
+
# e.g. training/serving a model that was never defined.
|
|
19
|
+
class SemanticError < StandardError; end
|
|
20
|
+
|
|
21
|
+
# Raised when a deployment target can't host the generated app -- e.g. asking
|
|
22
|
+
# to deploy a Torch model server to Vercel's serverless runtime.
|
|
23
|
+
class DeployError < StandardError; end
|
|
24
|
+
|
|
25
|
+
# Turns Parslet's internal failure cause tree into a human-friendly
|
|
26
|
+
# Aura::ParseError with line/column information and a source snippet.
|
|
27
|
+
module Diagnostics
|
|
28
|
+
module_function
|
|
29
|
+
|
|
30
|
+
# @param error [Parslet::ParseFailed]
|
|
31
|
+
# @param source [String] the original source text
|
|
32
|
+
# @return [Aura::ParseError]
|
|
33
|
+
def from_parslet(error, source)
|
|
34
|
+
cause = deepest_cause(error.parse_failure_cause)
|
|
35
|
+
line, column = location_for(cause)
|
|
36
|
+
message = +"Parse error"
|
|
37
|
+
message << " at line #{line}, column #{column}" if line
|
|
38
|
+
message << ": #{cause.to_s}" if cause
|
|
39
|
+
snippet = snippet_for(source, line, column)
|
|
40
|
+
message << "\n\n#{snippet}" if snippet
|
|
41
|
+
ParseError.new(message, line: line, column: column)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Walk to the most specific (deepest) cause so the reported position points
|
|
45
|
+
# at the actual offending token rather than the top-level rule.
|
|
46
|
+
def deepest_cause(cause)
|
|
47
|
+
return nil unless cause
|
|
48
|
+
|
|
49
|
+
current = cause
|
|
50
|
+
while current.respond_to?(:children) && current.children && !current.children.empty?
|
|
51
|
+
# Prefer the child with the furthest source position.
|
|
52
|
+
current = current.children.max_by { |c| position_of(c) || -1 }
|
|
53
|
+
end
|
|
54
|
+
current
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def location_for(cause)
|
|
58
|
+
return [nil, nil] unless cause && cause.respond_to?(:source) && cause.source
|
|
59
|
+
|
|
60
|
+
pos = cause.respond_to?(:pos) ? cause.pos : nil
|
|
61
|
+
return [nil, nil] unless pos
|
|
62
|
+
|
|
63
|
+
line, column = cause.source.line_and_column(pos)
|
|
64
|
+
[line, column]
|
|
65
|
+
rescue StandardError
|
|
66
|
+
[nil, nil]
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def position_of(cause)
|
|
70
|
+
cause.respond_to?(:pos) && cause.pos ? cause.pos.bytepos : nil
|
|
71
|
+
rescue StandardError
|
|
72
|
+
nil
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def snippet_for(source, line, column)
|
|
76
|
+
return nil unless line && column
|
|
77
|
+
|
|
78
|
+
src_line = source.lines[line - 1]
|
|
79
|
+
return nil unless src_line
|
|
80
|
+
|
|
81
|
+
src_line = src_line.chomp
|
|
82
|
+
caret = "#{' ' * (column - 1)}^"
|
|
83
|
+
" #{src_line}\n #{caret}"
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
data/lib/aura/docker.rb
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
|
|
5
|
+
module Aura
|
|
6
|
+
# Generates production deployment assets next to an .aura file: the
|
|
7
|
+
# transpiled standalone Ruby app, a Dockerfile, and a .dockerignore. Backs
|
|
8
|
+
# the `aura deploy` command.
|
|
9
|
+
module Docker
|
|
10
|
+
module_function
|
|
11
|
+
|
|
12
|
+
def build(filename)
|
|
13
|
+
source = File.read(filename)
|
|
14
|
+
ruby_code = Aura.transpile(source)
|
|
15
|
+
torch_app = ruby_code.include?("Torch::") # ML app needs torch-rb + LibTorch
|
|
16
|
+
|
|
17
|
+
dir = File.dirname(filename)
|
|
18
|
+
base = File.basename(filename, ".aura")
|
|
19
|
+
app_file = File.join(dir, "#{base}.rb")
|
|
20
|
+
|
|
21
|
+
File.write(app_file, ruby_code)
|
|
22
|
+
File.write(File.join(dir, "Dockerfile"), dockerfile("#{base}.rb", torch: torch_app))
|
|
23
|
+
File.write(File.join(dir, ".dockerignore"), dockerignore)
|
|
24
|
+
|
|
25
|
+
puts "Generated deployment assets:"
|
|
26
|
+
puts " - #{app_file}"
|
|
27
|
+
puts " - #{File.join(dir, 'Dockerfile')}"
|
|
28
|
+
puts " - #{File.join(dir, '.dockerignore')}"
|
|
29
|
+
puts " Build with: docker build -t #{base} #{dir.empty? ? '.' : dir}"
|
|
30
|
+
puts " NOTE: Torch apps need LibTorch in the image -- see the Dockerfile comment." if torch_app
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def dockerfile(app_file, torch: false)
|
|
34
|
+
gems = "sinatra puma json dotenv"
|
|
35
|
+
gems += " torch-rb torchvision red-datasets" if torch
|
|
36
|
+
libtorch = +""
|
|
37
|
+
if torch
|
|
38
|
+
libtorch = <<~TORCH
|
|
39
|
+
|
|
40
|
+
# Torch apps need LibTorch (the native PyTorch C++ library). The simplest
|
|
41
|
+
# path is to base the image on one that already bundles it, or download it:
|
|
42
|
+
# ENV LIBTORCH /opt/libtorch
|
|
43
|
+
# RUN curl -L https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-latest.zip -o /tmp/lt.zip \\
|
|
44
|
+
# && unzip /tmp/lt.zip -d /opt && rm /tmp/lt.zip
|
|
45
|
+
# Then build torch-rb against it: gem install torch-rb -- --with-torch-dir=$LIBTORCH
|
|
46
|
+
TORCH
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
<<~DOCKER
|
|
50
|
+
# Generated by Aura v#{Aura::VERSION}
|
|
51
|
+
FROM ruby:3.3-slim
|
|
52
|
+
|
|
53
|
+
WORKDIR /app
|
|
54
|
+
|
|
55
|
+
RUN apt-get update -qq \\
|
|
56
|
+
&& apt-get install -y --no-install-recommends build-essential#{torch ? ' curl unzip' : ''} \\
|
|
57
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
58
|
+
#{libtorch}
|
|
59
|
+
RUN gem install #{gems}
|
|
60
|
+
|
|
61
|
+
COPY . .
|
|
62
|
+
|
|
63
|
+
ENV RACK_ENV=production
|
|
64
|
+
EXPOSE 3000
|
|
65
|
+
|
|
66
|
+
CMD ["ruby", "#{app_file}"]
|
|
67
|
+
DOCKER
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def dockerignore
|
|
71
|
+
<<~IGNORE
|
|
72
|
+
.git
|
|
73
|
+
*.aura
|
|
74
|
+
models/*.pth
|
|
75
|
+
data/
|
|
76
|
+
tmp/
|
|
77
|
+
*.log
|
|
78
|
+
IGNORE
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
data/lib/aura/emitter.rb
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Aura
|
|
4
|
+
# A tiny indentation-aware code builder. Replaces ad-hoc string
|
|
5
|
+
# concatenation in the code generator so emitted Ruby is consistently
|
|
6
|
+
# indented and blocks are hard to leave unbalanced.
|
|
7
|
+
#
|
|
8
|
+
# e = Emitter.new
|
|
9
|
+
# e.line "x = 1"
|
|
10
|
+
# e.block("def forward(x)") { e.line "x" }
|
|
11
|
+
# e.to_s
|
|
12
|
+
#
|
|
13
|
+
class Emitter
|
|
14
|
+
INDENT = " "
|
|
15
|
+
|
|
16
|
+
def initialize
|
|
17
|
+
@lines = []
|
|
18
|
+
@depth = 0
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Append a line (or several "\n"-separated lines) at the current indent.
|
|
22
|
+
def line(text = "")
|
|
23
|
+
text.to_s.split("\n", -1).each do |raw|
|
|
24
|
+
@lines << (raw.empty? ? "" : "#{INDENT * @depth}#{raw}")
|
|
25
|
+
end
|
|
26
|
+
self
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Append a blank line.
|
|
30
|
+
def blank
|
|
31
|
+
@lines << ""
|
|
32
|
+
self
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Append a comment line.
|
|
36
|
+
def comment(text)
|
|
37
|
+
line("# #{text}")
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Emit `header` then an indented body produced by the block, then `footer`
|
|
41
|
+
# (defaults to `end`). Used for classes, methods, and Ruby blocks.
|
|
42
|
+
def block(header, footer = "end")
|
|
43
|
+
line(header)
|
|
44
|
+
indent { yield self }
|
|
45
|
+
line(footer)
|
|
46
|
+
self
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Increase indentation for the duration of the block.
|
|
50
|
+
def indent
|
|
51
|
+
@depth += 1
|
|
52
|
+
yield self
|
|
53
|
+
ensure
|
|
54
|
+
@depth -= 1
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def to_s
|
|
58
|
+
"#{@lines.join("\n")}\n"
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
data/lib/aura/parser.rb
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "parslet"
|
|
4
|
+
|
|
5
|
+
module Aura
|
|
6
|
+
# PEG grammar for the Aura DSL, built on Parslet. The grammar is whitespace-
|
|
7
|
+
# and blank-line tolerant; full-line `#` comments are stripped (and replaced
|
|
8
|
+
# by blank lines, preserving line numbers) before parsing -- see
|
|
9
|
+
# Aura.preprocess in lib/aura.rb.
|
|
10
|
+
class Parser < Parslet::Parser
|
|
11
|
+
# ---- lexical -------------------------------------------------------------
|
|
12
|
+
rule(:sp) { match('[ \t]').repeat }
|
|
13
|
+
rule(:sp1) { match('[ \t]').repeat(1) }
|
|
14
|
+
rule(:nl) { str("\r\n") | str("\n") | str("\r") }
|
|
15
|
+
# End of a content line: trailing spaces then a newline (or end of input).
|
|
16
|
+
rule(:eol) { sp >> (nl | any.absent?) }
|
|
17
|
+
# Zero or more whitespace-only lines.
|
|
18
|
+
rule(:blank_lines) { (sp >> nl).repeat }
|
|
19
|
+
|
|
20
|
+
rule(:identifier) { match('[a-zA-Z_]') >> match('[a-zA-Z0-9_]').repeat }
|
|
21
|
+
# Double-quoted string; `\X` escape pairs are consumed as a unit so a `\"`
|
|
22
|
+
# does not terminate the literal (unescaped by the Transformer).
|
|
23
|
+
rule(:string) do
|
|
24
|
+
str('"') >> ((str("\\") >> any) | (str('"').absent? >> any)).repeat.as(:str) >> str('"')
|
|
25
|
+
end
|
|
26
|
+
# Integer/float with optional leading sign and scientific notation
|
|
27
|
+
# (e.g. `-5`, `0.001`, `1e-4`, `3.2E+5`).
|
|
28
|
+
rule(:number) do
|
|
29
|
+
(str("-").maybe >> match('[0-9]').repeat(1) >>
|
|
30
|
+
(str(".") >> match('[0-9]').repeat(1)).maybe >>
|
|
31
|
+
(match('[eE]') >> match('[-+]').maybe >> match('[0-9]').repeat(1)).maybe).as(:number)
|
|
32
|
+
end
|
|
33
|
+
rule(:symbol) { str(":") >> identifier.as(:sym) }
|
|
34
|
+
rule(:boolean) { (str("true") | str("false")).as(:bool) }
|
|
35
|
+
rule(:value) { string | number | boolean | symbol }
|
|
36
|
+
|
|
37
|
+
# Comma separator that tolerates surrounding spaces.
|
|
38
|
+
rule(:comma) { sp >> str(",") >> sp }
|
|
39
|
+
|
|
40
|
+
# ---- helpers -------------------------------------------------------------
|
|
41
|
+
# A `do ... end` block whose body is zero or more `line_atom`s (blank lines
|
|
42
|
+
# between them are skipped). The collected lines are captured under `key`.
|
|
43
|
+
def block_body(line_atom, key)
|
|
44
|
+
str("do") >> eol >>
|
|
45
|
+
(blank_lines >> line_atom).repeat.as(key) >>
|
|
46
|
+
blank_lines >> sp >> str("end")
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# ---- program -------------------------------------------------------------
|
|
50
|
+
rule(:program) { blank_lines >> (statement >> blank_lines).repeat >> sp }
|
|
51
|
+
rule(:statement) do
|
|
52
|
+
dataset_stmt | env_stmt | model_stmt | train_stmt |
|
|
53
|
+
evaluate_stmt | route_stmt | run_stmt
|
|
54
|
+
end
|
|
55
|
+
root :program
|
|
56
|
+
|
|
57
|
+
# ---- dataset -------------------------------------------------------------
|
|
58
|
+
rule(:dataset_stmt) do
|
|
59
|
+
str("dataset") >> sp1 >> string.as(:ds_name) >> sp1 >>
|
|
60
|
+
str("from") >> sp1 >> identifier.as(:ds_source) >> sp1 >> string.as(:ds_path) >>
|
|
61
|
+
(sp1 >> block_body(dataset_option, :ds_options)).maybe >> eol
|
|
62
|
+
end
|
|
63
|
+
rule(:dataset_option) do
|
|
64
|
+
sp >> identifier.as(:opt_key) >> sp1 >> value.as(:opt_value) >> eol
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# ---- environment ---------------------------------------------------------
|
|
68
|
+
rule(:env_stmt) do
|
|
69
|
+
str("environment") >> sp1 >> identifier.as(:env_name) >> sp1 >>
|
|
70
|
+
block_body(env_line, :env_body) >> eol
|
|
71
|
+
end
|
|
72
|
+
rule(:env_line) do
|
|
73
|
+
sp >> identifier.as(:env_key) >> sp1 >> value.as(:env_value) >> eol
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# ---- model ---------------------------------------------------------------
|
|
77
|
+
rule(:model_stmt) do
|
|
78
|
+
str("model") >> sp1 >> identifier.as(:model_name) >> sp1 >> (
|
|
79
|
+
(str("neural_network") >> sp1 >> block_body(model_line, :nn_body)) |
|
|
80
|
+
(str("from") >> sp1 >> identifier.as(:provider) >> sp1 >> string.as(:model_id) >>
|
|
81
|
+
(sp1 >> block_body(llm_option, :llm_body)).maybe) |
|
|
82
|
+
(str("transfer") >> sp1 >> str("from") >> sp1 >> symbol.as(:base_model) >>
|
|
83
|
+
(sp1 >> block_body(model_line, :nn_body)).maybe)
|
|
84
|
+
) >> eol
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Optional config inside `model x from openai "id" do ... end`.
|
|
88
|
+
rule(:llm_option) do
|
|
89
|
+
sp >> (str("end") >> eol).absent? >> (
|
|
90
|
+
(str("system") >> sp1 >> string.as(:llm_system)) |
|
|
91
|
+
(str("temperature") >> sp1 >> number.as(:llm_temperature)) |
|
|
92
|
+
(str("max_tokens") >> sp1 >> number.as(:llm_max_tokens))
|
|
93
|
+
) >> eol
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
rule(:model_line) do
|
|
97
|
+
sp >> (
|
|
98
|
+
m_input_shape | m_input_text | m_conv | m_maxpool | m_batchnorm |
|
|
99
|
+
m_flatten | m_embedding | m_lstm | m_gru | m_dense | m_dropout |
|
|
100
|
+
m_output | m_greeting | m_load | m_save | m_freeze | m_unfreeze
|
|
101
|
+
) >> eol
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
rule(:m_embedding) do
|
|
105
|
+
str("layer") >> sp1 >> str("embedding") >> sp1 >>
|
|
106
|
+
str("vocab:") >> sp1 >> number.as(:embed_vocab) >> comma >>
|
|
107
|
+
str("dim:") >> sp1 >> number.as(:embed_dim)
|
|
108
|
+
end
|
|
109
|
+
rule(:m_lstm) do
|
|
110
|
+
str("layer") >> sp1 >> str("lstm") >> sp1 >> str("units:") >> sp1 >> number.as(:lstm_units)
|
|
111
|
+
end
|
|
112
|
+
rule(:m_gru) do
|
|
113
|
+
str("layer") >> sp1 >> str("gru") >> sp1 >> str("units:") >> sp1 >> number.as(:gru_units)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
rule(:m_input_shape) do
|
|
117
|
+
str("input") >> sp1 >> str("shape(") >> sp >>
|
|
118
|
+
(number >> comma.maybe).repeat(1).as(:input_shape) >> sp >> str(")") >>
|
|
119
|
+
(sp1 >> block_body(input_transform, :input_transforms)).maybe
|
|
120
|
+
end
|
|
121
|
+
# A preprocessing directive inside an `input shape(...) do ... end` block,
|
|
122
|
+
# e.g. `resize 28` or `to_tensor`. The `end`-line guard stops the value-less
|
|
123
|
+
# form from swallowing the block terminator.
|
|
124
|
+
rule(:input_transform) do
|
|
125
|
+
sp >> (str("end") >> eol).absent? >> identifier.as(:tf_key) >>
|
|
126
|
+
(sp1 >> value.as(:tf_value)).maybe >> eol
|
|
127
|
+
end
|
|
128
|
+
rule(:m_input_text) { str("input") >> sp1 >> str("text").as(:input_text) }
|
|
129
|
+
rule(:m_conv) do
|
|
130
|
+
str("layer") >> sp1 >> str("conv2d") >> sp1 >>
|
|
131
|
+
str("filters:") >> sp1 >> number.as(:conv_filters) >> comma >>
|
|
132
|
+
str("kernel:") >> sp1 >> number.as(:conv_kernel)
|
|
133
|
+
end
|
|
134
|
+
rule(:m_maxpool) do
|
|
135
|
+
str("layer") >> sp1 >> str("maxpool2d") >> sp1 >> str("size:") >> sp1 >> number.as(:pool_size)
|
|
136
|
+
end
|
|
137
|
+
rule(:m_batchnorm) { str("layer") >> sp1 >> str("batchnorm").as(:batchnorm) }
|
|
138
|
+
rule(:m_flatten) { str("layer") >> sp1 >> str("flatten").as(:flatten) }
|
|
139
|
+
rule(:m_dense) do
|
|
140
|
+
str("layer") >> sp1 >> str("dense") >> sp1 >> str("units:") >> sp1 >> number.as(:dense_units) >>
|
|
141
|
+
(comma >> str("activation:") >> sp1 >> symbol.as(:dense_activation)).maybe
|
|
142
|
+
end
|
|
143
|
+
rule(:m_dropout) do
|
|
144
|
+
str("layer") >> sp1 >> str("dropout") >> sp1 >> str("rate:") >> sp1 >> number.as(:dropout_rate)
|
|
145
|
+
end
|
|
146
|
+
rule(:m_output) do
|
|
147
|
+
str("output") >> sp1 >> str("units:") >> sp1 >> number.as(:out_units) >>
|
|
148
|
+
comma >> str("activation:") >> sp1 >> symbol.as(:out_activation)
|
|
149
|
+
end
|
|
150
|
+
rule(:m_greeting) do
|
|
151
|
+
str("output") >> sp1 >> str("greeting") >> sp1 >> string.as(:greeting)
|
|
152
|
+
end
|
|
153
|
+
rule(:m_save) do
|
|
154
|
+
str("save") >> sp1 >> str("weights") >> sp1 >> str("to") >> sp1 >> string.as(:save_path)
|
|
155
|
+
end
|
|
156
|
+
rule(:m_load) do
|
|
157
|
+
str("load") >> sp1 >> str("weights") >> sp1 >> str("from") >> sp1 >> string.as(:load_path)
|
|
158
|
+
end
|
|
159
|
+
rule(:m_freeze) do
|
|
160
|
+
str("freeze") >> sp1 >> str("until") >> sp1 >> symbol.as(:freeze_until)
|
|
161
|
+
end
|
|
162
|
+
rule(:m_unfreeze) { str("unfreeze") >> sp1 >> str("all").as(:unfreeze_all) }
|
|
163
|
+
|
|
164
|
+
# ---- train ---------------------------------------------------------------
|
|
165
|
+
rule(:train_stmt) do
|
|
166
|
+
str("train") >> sp1 >> identifier.as(:tr_model) >> sp1 >> str("on") >> sp1 >>
|
|
167
|
+
string.as(:tr_dataset) >> sp1 >> block_body(train_option, :tr_body) >> eol
|
|
168
|
+
end
|
|
169
|
+
rule(:train_option) do
|
|
170
|
+
sp >> (
|
|
171
|
+
(str("epochs") >> sp1 >> number.as(:epochs)) |
|
|
172
|
+
(str("batch_size") >> sp1 >> number.as(:batch_size)) |
|
|
173
|
+
(str("optimizer") >> sp1 >> symbol.as(:optimizer) >>
|
|
174
|
+
(comma >> str("learning_rate:") >> sp1 >> number.as(:lr)).maybe) |
|
|
175
|
+
(str("scheduler") >> sp1 >> symbol.as(:scheduler)) |
|
|
176
|
+
(str("loss") >> sp1 >> symbol.as(:loss)) |
|
|
177
|
+
(str("metrics") >> sp1 >> symbol.as(:metrics))
|
|
178
|
+
) >> eol
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# ---- evaluate ------------------------------------------------------------
|
|
182
|
+
rule(:evaluate_stmt) do
|
|
183
|
+
str("evaluate") >> sp1 >> identifier.as(:ev_model) >> sp1 >> str("on") >> sp1 >>
|
|
184
|
+
string.as(:ev_dataset) >> eol
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# ---- route ---------------------------------------------------------------
|
|
188
|
+
rule(:route_stmt) do
|
|
189
|
+
str("route") >> sp1 >> string.as(:rt_path) >> sp1 >> identifier.as(:rt_method) >> sp1 >>
|
|
190
|
+
block_body(route_line, :rt_body) >> eol
|
|
191
|
+
end
|
|
192
|
+
rule(:route_line) do
|
|
193
|
+
sp >> (route_output | route_auth) >> eol
|
|
194
|
+
end
|
|
195
|
+
rule(:route_output) do
|
|
196
|
+
str("output prediction from") >> sp1 >> identifier.as(:route_model) >>
|
|
197
|
+
str(".predict(") >> identifier.as(:route_input) >> str(")") >>
|
|
198
|
+
(sp1 >> str("as") >> sp1 >> symbol.as(:route_as)).maybe >>
|
|
199
|
+
(sp1 >> str("format") >> sp1 >> symbol.as(:route_format)).maybe
|
|
200
|
+
end
|
|
201
|
+
rule(:route_auth) do
|
|
202
|
+
str("authenticate with") >> sp1 >> symbol.as(:auth)
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# ---- run -----------------------------------------------------------------
|
|
206
|
+
rule(:run_stmt) { str("run web on port:") >> sp1 >> number.as(:port) >> eol }
|
|
207
|
+
end
|
|
208
|
+
end
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "parslet"
|
|
4
|
+
|
|
5
|
+
module Aura
|
|
6
|
+
# Pure helper functions for assembling semantic node hashes. Kept in a module
|
|
7
|
+
# (rather than Transformer instance methods) because Parslet::Transform
|
|
8
|
+
# evaluates rule blocks in their own context where instance methods aren't in
|
|
9
|
+
# scope -- constants like `Aura::Nodes` always are.
|
|
10
|
+
module Nodes
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
def list(value)
|
|
14
|
+
value.is_a?(Array) ? value : [value]
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Keep only real node hashes, discarding whitespace slices that Parslet may
|
|
18
|
+
# leave in a repeated body.
|
|
19
|
+
def clean(body)
|
|
20
|
+
list(body).select { |x| x.is_a?(Hash) }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def model(name, body)
|
|
24
|
+
layers = clean(body)
|
|
25
|
+
if layers.any? { |l| %i[greeting input_text].include?(l[:type]) }
|
|
26
|
+
greeting = layers.find { |l| l[:type] == :greeting }
|
|
27
|
+
{ type: :model, kind: :text, name: name.to_s, layers: layers,
|
|
28
|
+
greeting: (greeting && greeting[:text]) || "Hello from Aura!" }
|
|
29
|
+
else
|
|
30
|
+
{ type: :model, kind: :torch, name: name.to_s, layers: layers, torch_model: true }
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def transfer(name, base, body)
|
|
35
|
+
{ type: :model, kind: :transfer, name: name.to_s, base_model: base, layers: clean(body) }
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def llm(name, provider, model_id, body)
|
|
39
|
+
cfg = clean(body).each_with_object({}) { |h, acc| acc.merge!(h) }
|
|
40
|
+
{ type: :model, kind: :llm, name: name.to_s, provider: provider.to_s.to_sym,
|
|
41
|
+
model_id: model_id.to_s, system: cfg[:system],
|
|
42
|
+
temperature: cfg[:temperature], max_tokens: cfg[:max_tokens] }
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def train(model, dataset, body)
|
|
46
|
+
config = clean(body).each_with_object({}) { |opt, h| h.merge!(opt) }
|
|
47
|
+
{ type: :train, model: model.to_s, dataset: dataset.to_s, config: config }
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def route(path, method, body)
|
|
51
|
+
lines = clean(body)
|
|
52
|
+
out = lines.find { |l| l.key?(:route_model) }
|
|
53
|
+
auth = lines.find { |l| l.key?(:auth) }
|
|
54
|
+
{ type: :route, path: path.to_s, method: method.to_s,
|
|
55
|
+
model: out && out[:route_model],
|
|
56
|
+
input_var: (out && out[:route_input]) || "input",
|
|
57
|
+
format: out && out[:route_format],
|
|
58
|
+
postprocess: out && out[:route_as],
|
|
59
|
+
auth: auth && auth[:auth] }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def settings(body)
|
|
63
|
+
clean(body).each_with_object({}) { |kv, h| h.merge!(kv) }
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def dataset(name, source, path, options)
|
|
67
|
+
{ type: :dataset, name: name.to_s, source: source.to_s, path: path.to_s,
|
|
68
|
+
options: settings(options) }
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Walks the raw Parslet parse tree and rewrites it into a flat list of
|
|
73
|
+
# semantic node hashes, each tagged with a :type the code generator switches
|
|
74
|
+
# on. Numeric leaves are coerced to Integer/Float here (not String) so node
|
|
75
|
+
# values are directly usable.
|
|
76
|
+
class Transformer < Parslet::Transform
|
|
77
|
+
# ---- leaves --------------------------------------------------------------
|
|
78
|
+
rule(str: simple(:s)) do
|
|
79
|
+
s.to_s.gsub(/\\(.)/) do
|
|
80
|
+
case Regexp.last_match(1)
|
|
81
|
+
when "n" then "\n"
|
|
82
|
+
when "t" then "\t"
|
|
83
|
+
else Regexp.last_match(1) # \" -> ", \\ -> \, and any other \X -> X
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
rule(sym: simple(:s)) { s.to_s.to_sym }
|
|
88
|
+
rule(bool: simple(:b)) { b.to_s == "true" }
|
|
89
|
+
# Float when it has a decimal point or exponent (1e-4); Integer otherwise
|
|
90
|
+
# (including signed values like -5).
|
|
91
|
+
rule(number: simple(:n)) { (x = n.to_s).match?(/[.eE]/) ? x.to_f : x.to_i }
|
|
92
|
+
|
|
93
|
+
# ---- model body lines ----------------------------------------------------
|
|
94
|
+
rule(input_shape: subtree(:s)) { { type: :input, shape: Aura::Nodes.list(s) } }
|
|
95
|
+
rule(input_shape: subtree(:s), input_transforms: subtree(:t)) { { type: :input, shape: Aura::Nodes.list(s), transforms: Aura::Nodes.clean(t) } }
|
|
96
|
+
rule(tf_key: simple(:k), tf_value: simple(:v)) { { transform: k.to_s.to_sym, value: v } }
|
|
97
|
+
rule(tf_key: simple(:k)) { { transform: k.to_s.to_sym } }
|
|
98
|
+
rule(input_text: simple(:_x)) { { type: :input_text } }
|
|
99
|
+
rule(conv_filters: simple(:f), conv_kernel: simple(:k)) { { type: :conv2d, filters: f.to_i, kernel: k.to_i } }
|
|
100
|
+
rule(pool_size: simple(:s)) { { type: :maxpool2d, size: s.to_i } }
|
|
101
|
+
rule(batchnorm: simple(:_x)) { { type: :batchnorm } }
|
|
102
|
+
rule(flatten: simple(:_x)) { { type: :flatten } }
|
|
103
|
+
rule(dense_units: simple(:u), dense_activation: simple(:a)) { { type: :dense, units: u.to_i, activation: a } }
|
|
104
|
+
rule(dense_units: simple(:u)) { { type: :dense, units: u.to_i, activation: :linear } }
|
|
105
|
+
rule(dropout_rate: simple(:r)) { { type: :dropout, rate: r.to_f } }
|
|
106
|
+
rule(out_units: simple(:u), out_activation: simple(:a)) { { type: :output, units: u.to_i, activation: a } }
|
|
107
|
+
rule(greeting: simple(:g)) { { type: :greeting, text: g.to_s } }
|
|
108
|
+
rule(save_path: simple(:p)) { { type: :save_weights, path: p.to_s } }
|
|
109
|
+
rule(load_path: simple(:p)) { { type: :load_weights, path: p.to_s } }
|
|
110
|
+
rule(freeze_until: simple(:l)) { { type: :freeze, until: l } }
|
|
111
|
+
rule(unfreeze_all: simple(:_x)) { { type: :unfreeze_all } }
|
|
112
|
+
rule(embed_vocab: simple(:v), embed_dim: simple(:d)) { { type: :embedding, vocab: v.to_i, dim: d.to_i } }
|
|
113
|
+
rule(lstm_units: simple(:u)) { { type: :lstm, units: u.to_i } }
|
|
114
|
+
rule(gru_units: simple(:u)) { { type: :gru, units: u.to_i } }
|
|
115
|
+
|
|
116
|
+
# ---- LLM config body lines -----------------------------------------------
|
|
117
|
+
rule(llm_system: simple(:s)) { { system: s.to_s } }
|
|
118
|
+
rule(llm_temperature: simple(:t)) { { temperature: t } }
|
|
119
|
+
rule(llm_max_tokens: simple(:m)) { { max_tokens: m.to_i } }
|
|
120
|
+
|
|
121
|
+
# ---- route body lines ----------------------------------------------------
|
|
122
|
+
rule(route_model: simple(:m), route_input: simple(:i), route_as: simple(:a), route_format: simple(:f)) { { route_model: m.to_s, route_input: i.to_s, route_as: a, route_format: f } }
|
|
123
|
+
rule(route_model: simple(:m), route_input: simple(:i), route_as: simple(:a)) { { route_model: m.to_s, route_input: i.to_s, route_as: a, route_format: nil } }
|
|
124
|
+
rule(route_model: simple(:m), route_input: simple(:i), route_format: simple(:f)) { { route_model: m.to_s, route_input: i.to_s, route_format: f } }
|
|
125
|
+
rule(route_model: simple(:m), route_input: simple(:i)) { { route_model: m.to_s, route_input: i.to_s, route_format: nil } }
|
|
126
|
+
rule(auth: simple(:a)) { { auth: a } }
|
|
127
|
+
|
|
128
|
+
# ---- key/value lines (env + dataset options) -----------------------------
|
|
129
|
+
rule(env_key: simple(:k), env_value: simple(:v)) { { k.to_s.to_sym => v } }
|
|
130
|
+
rule(opt_key: simple(:k), opt_value: simple(:v)) { { k.to_s.to_sym => v } }
|
|
131
|
+
|
|
132
|
+
# ---- statements ----------------------------------------------------------
|
|
133
|
+
rule(ds_name: simple(:n), ds_source: simple(:s), ds_path: simple(:p), ds_options: subtree(:o)) { Aura::Nodes.dataset(n, s, p, o) }
|
|
134
|
+
rule(ds_name: simple(:n), ds_source: simple(:s), ds_path: simple(:p)) { Aura::Nodes.dataset(n, s, p, []) }
|
|
135
|
+
|
|
136
|
+
rule(env_name: simple(:n), env_body: subtree(:b)) { { type: :environment, name: n.to_s, settings: Aura::Nodes.settings(b) } }
|
|
137
|
+
|
|
138
|
+
rule(model_name: simple(:n), nn_body: subtree(:b)) { Aura::Nodes.model(n, b) }
|
|
139
|
+
rule(model_name: simple(:n), provider: simple(:pr), model_id: simple(:mid)) { Aura::Nodes.llm(n, pr, mid, []) }
|
|
140
|
+
rule(model_name: simple(:n), provider: simple(:pr), model_id: simple(:mid), llm_body: subtree(:b)) { Aura::Nodes.llm(n, pr, mid, b) }
|
|
141
|
+
rule(model_name: simple(:n), base_model: simple(:bm)) { { type: :model, kind: :transfer, name: n.to_s, base_model: bm, layers: [] } }
|
|
142
|
+
rule(model_name: simple(:n), base_model: simple(:bm), nn_body: subtree(:b)) { Aura::Nodes.transfer(n, bm, b) }
|
|
143
|
+
|
|
144
|
+
rule(tr_model: simple(:m), tr_dataset: simple(:d), tr_body: subtree(:b)) { Aura::Nodes.train(m, d, b) }
|
|
145
|
+
|
|
146
|
+
rule(ev_model: simple(:m), ev_dataset: simple(:d)) { { type: :evaluate, model: m.to_s, dataset: d.to_s } }
|
|
147
|
+
|
|
148
|
+
rule(rt_path: simple(:p), rt_method: simple(:m), rt_body: subtree(:b)) { Aura::Nodes.route(p, m, b) }
|
|
149
|
+
|
|
150
|
+
rule(port: simple(:p)) { { type: :run_web, port: p.to_i } }
|
|
151
|
+
end
|
|
152
|
+
end
|
data/lib/aura/vercel.rb
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
|
|
5
|
+
module Aura
|
|
6
|
+
# Generates Vercel deployment assets for an .aura file whose generated app is
|
|
7
|
+
# a stateless Rack app -- i.e. LLM-only (no Torch). Torch model servers need
|
|
8
|
+
# LibTorch and a long-lived process, which exceed Vercel's serverless limits
|
|
9
|
+
# (function size, no GPU, short timeouts); for those use `aura deploy`
|
|
10
|
+
# (Dockerfile) with a container host. Backs `aura deploy <file> --target vercel`.
|
|
11
|
+
module Vercel
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
def build(filename)
|
|
15
|
+
nodes = Aura.to_nodes(File.read(filename))
|
|
16
|
+
refuse_torch!(filename) if uses_torch?(nodes)
|
|
17
|
+
ruby_code = Aura::CodeGen.generate(nodes)
|
|
18
|
+
|
|
19
|
+
dir = File.dirname(filename)
|
|
20
|
+
base = File.basename(filename, ".aura")
|
|
21
|
+
api_dir = File.join(dir, "api")
|
|
22
|
+
FileUtils.mkdir_p(api_dir)
|
|
23
|
+
|
|
24
|
+
app_file = File.join(dir, "#{base}.rb")
|
|
25
|
+
index_file = File.join(api_dir, "index.rb")
|
|
26
|
+
json_file = File.join(dir, "vercel.json")
|
|
27
|
+
|
|
28
|
+
File.write(app_file, ruby_code)
|
|
29
|
+
File.write(index_file, handler(base))
|
|
30
|
+
File.write(json_file, vercel_json)
|
|
31
|
+
|
|
32
|
+
puts "Generated Vercel assets (LLM-only app):"
|
|
33
|
+
puts " - #{app_file}"
|
|
34
|
+
puts " - #{index_file}"
|
|
35
|
+
puts " - #{json_file}"
|
|
36
|
+
puts " Set secrets in the Vercel dashboard (e.g. OPENAI_API_KEY, AURA_API_TOKEN),"
|
|
37
|
+
puts " then deploy with: vercel deploy"
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Torch is in play when a model is a torch/transfer net, or when the program
|
|
41
|
+
# trains/evaluates (which pulls in LibTorch + datasets).
|
|
42
|
+
def uses_torch?(nodes)
|
|
43
|
+
nodes.any? do |n|
|
|
44
|
+
(n[:type] == :model && %i[torch transfer].include?(n[:kind])) ||
|
|
45
|
+
%i[train evaluate].include?(n[:type])
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def refuse_torch!(filename)
|
|
50
|
+
raise DeployError,
|
|
51
|
+
"This app uses Torch, which exceeds Vercel's serverless limits (LibTorch " \
|
|
52
|
+
"size, no GPU, short timeouts). Use `aura deploy #{filename}` (Dockerfile) " \
|
|
53
|
+
"with a container host (Fly.io / Render / Cloud Run) instead."
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Vercel's Ruby runtime serves a `Handler` Rack app from api/*.rb. We load
|
|
57
|
+
# the transpiled app with the classic-Sinatra boot disabled and expose its
|
|
58
|
+
# Rack application object.
|
|
59
|
+
def handler(base)
|
|
60
|
+
<<~RUBY
|
|
61
|
+
# Generated by Aura v#{Aura::VERSION} -- Vercel Ruby serverless entrypoint.
|
|
62
|
+
ENV["RACK_ENV"] ||= "production"
|
|
63
|
+
require_relative "../#{base}"
|
|
64
|
+
|
|
65
|
+
# Do not boot a listening server inside the serverless function; Vercel
|
|
66
|
+
# invokes the Rack app directly.
|
|
67
|
+
Sinatra::Application.set(:run, false)
|
|
68
|
+
|
|
69
|
+
Handler = Sinatra::Application
|
|
70
|
+
RUBY
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Uses the long-standing `@vercel/ruby` builder. Ruby is a community runtime
|
|
74
|
+
# on Vercel; adjust the builder for your account if the identifier changes.
|
|
75
|
+
def vercel_json
|
|
76
|
+
<<~JSON
|
|
77
|
+
{
|
|
78
|
+
"builds": [
|
|
79
|
+
{ "src": "api/index.rb", "use": "@vercel/ruby" }
|
|
80
|
+
],
|
|
81
|
+
"routes": [
|
|
82
|
+
{ "src": "/(.*)", "dest": "/api/index.rb" }
|
|
83
|
+
]
|
|
84
|
+
}
|
|
85
|
+
JSON
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|