kward 0.84.0 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +164 -0
- data/CONTRIBUTING.md +1 -1
- data/Gemfile.lock +6 -2
- data/README.md +5 -5
- data/Rakefile +3 -0
- data/doc/agent-tools.md +24 -1
- data/doc/api.md +20 -2
- data/doc/composer.md +5 -4
- data/doc/configuration.md +86 -14
- data/doc/extensibility.md +4 -1
- data/doc/getting-started.md +3 -3
- data/doc/git.md +4 -2
- data/doc/memory.md +1 -1
- data/doc/pan.md +6 -0
- data/doc/permissions.md +4 -4
- data/doc/plugins.md +504 -20
- data/doc/providers.md +10 -4
- data/doc/rpc.md +279 -23
- data/doc/sandboxing.md +11 -5
- data/doc/security.md +7 -1
- data/doc/session-management.md +41 -5
- data/doc/shell.md +643 -27
- data/doc/tabs.md +8 -3
- data/doc/transports.md +15 -0
- data/doc/troubleshooting.md +1 -1
- data/doc/usage.md +25 -12
- data/doc/workspace-tools.md +9 -0
- data/examples/plugins/openrouter_proxy/README.md +103 -0
- data/examples/plugins/openrouter_proxy/fake_embeddings.rb +85 -0
- data/examples/plugins/openrouter_proxy/plugin.rb +7 -0
- data/examples/plugins/openrouter_proxy/protocol.rb +202 -0
- data/examples/plugins/openrouter_proxy/server.rb +186 -0
- data/examples/plugins/space_invaders.rb +1 -2
- data/examples/plugins/stardate_footer.rb +2 -2
- data/examples/plugins/telegram/plugin.rb +1 -1
- data/exe/kward +2 -2
- data/exe/kwsh +7 -0
- data/kward.gemspec +3 -1
- data/lib/kward/agent.rb +66 -19
- data/lib/kward/cli/commands.rb +41 -11
- data/lib/kward/cli/compaction.rb +9 -3
- data/lib/kward/cli/interactive_turn.rb +10 -0
- data/lib/kward/cli/plugins.rb +85 -12
- data/lib/kward/cli/prompt_interface.rb +29 -6
- data/lib/kward/cli/rendering.rb +24 -28
- data/lib/kward/cli/runtime_helpers.rb +168 -56
- data/lib/kward/cli/sessions.rb +9 -5
- data/lib/kward/cli/settings/menus.rb +745 -0
- data/lib/kward/cli/settings/model.rb +316 -0
- data/lib/kward/cli/settings.rb +6 -1055
- data/lib/kward/cli/slash_commands.rb +45 -4
- data/lib/kward/cli/tabs.rb +257 -44
- data/lib/kward/cli/tool_summaries.rb +2 -0
- data/lib/kward/{cli_transcript_formatter.rb → cli/transcript_formatter.rb} +6 -13
- data/lib/kward/cli/worktrees.rb +66 -2
- data/lib/kward/cli.rb +82 -31
- data/lib/kward/compactor.rb +18 -7
- data/lib/kward/config/core.rb +400 -0
- data/lib/kward/config/extensions.rb +99 -0
- data/lib/kward/config/paths.rb +11 -0
- data/lib/kward/config/prompts.rb +313 -0
- data/lib/kward/config/settings.rb +257 -0
- data/lib/kward/config_files.rb +14 -1008
- data/lib/kward/conversation.rb +41 -2
- data/lib/kward/entrypoint.rb +14 -0
- data/lib/kward/events.rb +2 -0
- data/lib/kward/generated_images.rb +237 -0
- data/lib/kward/image_attachments.rb +1 -1
- data/lib/kward/model/catalog.rb +6 -3
- data/lib/kward/model/client.rb +107 -38
- data/lib/kward/model/codex_images.rb +83 -0
- data/lib/kward/model/copilot_models.rb +10 -6
- data/lib/kward/model/model_info.rb +42 -7
- data/lib/kward/{openrouter_model_cache.rb → model/openrouter_model_cache.rb} +7 -5
- data/lib/kward/model/payloads.rb +18 -8
- data/lib/kward/model/sources.rb +3 -1
- data/lib/kward/model/stream_parser.rb +4 -7
- data/lib/kward/model/typesafe_client.rb +78 -0
- data/lib/kward/pan/index.html.erb +8 -1
- data/lib/kward/pan/server.rb +23 -8
- data/lib/kward/permissions/policy.rb +6 -2
- data/lib/kward/plugin_registry.rb +2 -659
- data/lib/kward/plugins/actions.rb +453 -0
- data/lib/kward/plugins/chat_contract.rb +121 -0
- data/lib/kward/{plugin_chat_runtime.rb → plugins/chat_runtime.rb} +62 -18
- data/lib/kward/plugins/codex_responses.rb +73 -0
- data/lib/kward/plugins/host.rb +248 -0
- data/lib/kward/plugins/http_server.rb +83 -0
- data/lib/kward/plugins/registry.rb +1229 -0
- data/lib/kward/plugins/resources.rb +206 -0
- data/lib/kward/plugins/turn_request.rb +36 -0
- data/lib/kward/plugins/ui.rb +219 -0
- data/lib/kward/prompt_interface/composer_controller.rb +0 -1
- data/lib/kward/prompt_interface/composer_renderer.rb +160 -73
- data/lib/kward/prompt_interface/composer_state.rb +1 -1
- data/lib/kward/prompt_interface/editor/controller.rb +2 -2
- data/lib/kward/prompt_interface/editor/renderer.rb +10 -7
- data/lib/kward/prompt_interface/editor/runner.rb +1 -1
- data/lib/kward/prompt_interface/editor/state.rb +1 -1
- data/lib/kward/prompt_interface/file_overlay.rb +3 -3
- data/lib/kward/prompt_interface/generated_image_preview.rb +162 -0
- data/lib/kward/prompt_interface/git_prompt.rb +18 -11
- data/lib/kward/prompt_interface/interactive/renderer.rb +1 -1
- data/lib/kward/prompt_interface/interactive/state.rb +18 -20
- data/lib/kward/prompt_interface/key_handler.rb +108 -25
- data/lib/kward/prompt_interface/layout.rb +11 -1
- data/lib/kward/prompt_interface/overlay_renderer.rb +26 -9
- data/lib/kward/prompt_interface/plugin_ui_requests.rb +82 -0
- data/lib/kward/prompt_interface/project_browser.rb +27 -54
- data/lib/kward/prompt_interface/prompt_renderer.rb +27 -35
- data/lib/kward/prompt_interface/question_prompt.rb +19 -28
- data/lib/kward/prompt_interface/runtime_state.rb +37 -10
- data/lib/kward/prompt_interface/screen.rb +30 -20
- data/lib/kward/prompt_interface/selection_prompt.rb +8 -8
- data/lib/kward/prompt_interface/slash_overlay.rb +1 -1
- data/lib/kward/prompt_interface/transcript_renderer.rb +7 -6
- data/lib/kward/prompt_interface.rb +149 -30
- data/lib/kward/prompts/commands.rb +2 -1
- data/lib/kward/prompts.rb +7 -1
- data/lib/kward/{adaptive_pty_output_sink.rb → pty/adaptive_output_sink.rb} +39 -5
- data/lib/kward/{interactive_pty_runner.rb → pty/interactive_runner.rb} +22 -30
- data/lib/kward/{local_command_runner.rb → pty/local_command_runner.rb} +1 -1
- data/lib/kward/{local_pty_command_runner.rb → pty/local_pty_runner.rb} +3 -9
- data/lib/kward/{pty_output_sink.rb → pty/output_sink.rb} +9 -4
- data/lib/kward/pty/transcript_normalizer.rb +137 -0
- data/lib/kward/rpc/config_manager.rb +3 -0
- data/lib/kward/rpc/plugin_chat_manager.rb +30 -10
- data/lib/kward/rpc/prompt_bridge.rb +25 -0
- data/lib/kward/rpc/server.rb +118 -17
- data/lib/kward/rpc/session_manager.rb +208 -60
- data/lib/kward/rpc/session_tree_rows.rb +2 -2
- data/lib/kward/rpc/tool_metadata.rb +1 -1
- data/lib/kward/rpc/transcript_normalizer.rb +11 -7
- data/lib/kward/sandbox/command_runner.rb +1 -1
- data/lib/kward/{session_catalog.rb → sessions/catalog.rb} +1 -1
- data/lib/kward/{session_store.rb → sessions/store.rb} +39 -12
- data/lib/kward/{session_tree_nodes.rb → sessions/tree_nodes.rb} +3 -3
- data/lib/kward/{session_tree_renderer.rb → sessions/tree_renderer.rb} +4 -4
- data/lib/kward/{session_tree_tool_display.rb → sessions/tree_tool_display.rb} +1 -1
- data/lib/kward/{kwsh.rb → shell/kwsh.rb} +30 -10
- data/lib/kward/{kwshrc.rb → shell/kwshrc.rb} +11 -2
- data/lib/kward/shell/native_arithmetic.rb +255 -0
- data/lib/kward/shell/native_arrays.rb +102 -0
- data/lib/kward/shell/native_braces.rb +96 -0
- data/lib/kward/shell/native_builtins.rb +476 -0
- data/lib/kward/shell/native_completion.rb +169 -0
- data/lib/kward/shell/native_condition.rb +75 -0
- data/lib/kward/shell/native_directory.rb +116 -0
- data/lib/kward/shell/native_exec_state.rb +73 -0
- data/lib/kward/shell/native_expansion.rb +252 -0
- data/lib/kward/shell/native_fields.rb +62 -0
- data/lib/kward/shell/native_glob.rb +96 -0
- data/lib/kward/shell/native_history.rb +242 -0
- data/lib/kward/shell/native_interactive.rb +164 -0
- data/lib/kward/shell/native_invocation.rb +42 -0
- data/lib/kward/shell/native_jobs.rb +250 -0
- data/lib/kward/shell/native_line_editor.rb +382 -0
- data/lib/kward/shell/native_line_terminal.rb +286 -0
- data/lib/kward/shell/native_locale.rb +109 -0
- data/lib/kward/shell/native_locals.rb +57 -0
- data/lib/kward/shell/native_pattern.rb +200 -0
- data/lib/kward/shell/native_process.rb +279 -0
- data/lib/kward/shell/native_program.rb +40 -0
- data/lib/kward/shell/native_programmable_completion.rb +115 -0
- data/lib/kward/shell/native_prompt_format.rb +89 -0
- data/lib/kward/shell/native_runtime.rb +964 -0
- data/lib/kward/shell/native_substitution.rb +110 -0
- data/lib/kward/shell/native_syntax.rb +946 -0
- data/lib/kward/shell/native_traps.rb +196 -0
- data/lib/kward/shell/native_vi.rb +407 -0
- data/lib/kward/shell/native_vi_terminal.rb +19 -0
- data/lib/kward/shell/native_worker.rb +82 -0
- data/lib/kward/{persistent_shell_session.rb → shell/persistent_session.rb} +187 -193
- data/lib/kward/shell/prompt.rb +66 -0
- data/lib/kward/shell/prompt_agent.rb +17 -0
- data/lib/kward/{shell_prompt_session.rb → shell/prompt_session.rb} +1 -1
- data/lib/kward/shell/script_runner.rb +244 -0
- data/lib/kward/shell/standalone_agent.rb +31 -0
- data/lib/kward/shell/standalone_link.rb +78 -0
- data/lib/kward/shell/standalone_terminal.rb +287 -0
- data/lib/kward/skills/trust_store.rb +1 -1
- data/lib/kward/tabs/driver.rb +194 -0
- data/lib/kward/{tab_store.rb → tabs/store.rb} +2 -2
- data/lib/kward/{clipboard.rb → terminal/clipboard.rb} +1 -1
- data/lib/kward/{terminal_image_support.rb → terminal/image_support.rb} +1 -1
- data/lib/kward/{terminal_keys.rb → terminal/keys.rb} +22 -1
- data/lib/kward/terminal/locale_text.rb +29 -0
- data/lib/kward/{terminal_sequences.rb → terminal/sequences.rb} +45 -4
- data/lib/kward/tools/base.rb +18 -0
- data/lib/kward/tools/context_for_task.rb +15 -6
- data/lib/kward/tools/datetime.rb +85 -0
- data/lib/kward/tools/edit_file.rb +9 -6
- data/lib/kward/tools/generate_image.rb +36 -0
- data/lib/kward/tools/git_commit.rb +13 -7
- data/lib/kward/tools/list_directory.rb +4 -4
- data/lib/kward/tools/plugin_tool.rb +41 -0
- data/lib/kward/tools/prepare_shell_command.rb +1 -1
- data/lib/kward/tools/read_file.rb +7 -6
- data/lib/kward/tools/registry.rb +130 -21
- data/lib/kward/tools/run_shell_command.rb +10 -8
- data/lib/kward/tools/search/code.rb +1 -1
- data/lib/kward/tools/summarize_file_structure.rb +5 -5
- data/lib/kward/tools/tool_call.rb +4 -1
- data/lib/kward/tools/typesafe_evaluate.rb +81 -0
- data/lib/kward/tools/workspace_targets.rb +58 -0
- data/lib/kward/tools/write_file.rb +9 -6
- data/lib/kward/{export_path.rb → transcripts/export_path.rb} +1 -1
- data/lib/kward/{markdown_transcript.rb → transcripts/markdown_transcript.rb} +10 -2
- data/lib/kward/transcripts/turn_summary_formatter.rb +93 -0
- data/lib/kward/transport/contracts.rb +200 -0
- data/lib/kward/transport/gateway.rb +81 -36
- data/lib/kward/transport/plugin_chat_gateway.rb +3 -2
- data/lib/kward/transport.rb +1 -200
- data/lib/kward/turn_summary.rb +68 -0
- data/lib/kward/version.rb +1 -1
- data/lib/kward/{workspace_factory.rb → workspace/factory.rb} +2 -2
- data/lib/kward/{git_worktree_manager.rb → workspace/git_worktree_manager.rb} +28 -0
- data/lib/kward/{workspace.rb → workspace/workspace.rb} +3 -3
- data/lib/main.rb +6 -3
- data/templates/default/fulldoc/html/css/kward.css +125 -0
- data/templates/default/fulldoc/html/images/kward_screen_1.png +0 -0
- data/templates/default/fulldoc/html/setup.rb +1 -1
- data/templates/default/layout/html/layout.erb +16 -4
- metadata +150 -48
- data/lib/kward/pty_transcript_normalizer.rb +0 -93
- data/lib/kward/shell_prompt.rb +0 -52
- data/lib/kward/tab_driver.rb +0 -90
- data/templates/default/fulldoc/html/images/kward_workflow.svg +0 -52
- /data/lib/kward/{editor_prompt.rb → cli/editor_prompt.rb} +0 -0
- /data/lib/kward/{editor_prompt_session.rb → cli/editor_prompt_session.rb} +0 -0
- /data/lib/kward/{diff_view_mode.rb → prompt_interface/editor/diff_view_mode.rb} +0 -0
- /data/lib/kward/{editor_mode.rb → prompt_interface/editor/editor_mode.rb} +0 -0
- /data/lib/kward/{markdown_code_block.rb → prompt_interface/editor/markdown_code_block.rb} +0 -0
- /data/lib/kward/{scratchpad_languages.rb → prompt_interface/editor/scratchpad_languages.rb} +0 -0
- /data/lib/kward/{scratchpad_runner.rb → prompt_interface/editor/scratchpad_runner.rb} +0 -0
- /data/lib/kward/{detached_run.rb → pty/detached_run.rb} +0 -0
- /data/lib/kward/{session_diff.rb → sessions/diff.rb} +0 -0
- /data/lib/kward/{session_naming.rb → sessions/naming.rb} +0 -0
- /data/lib/kward/{session_trash.rb → sessions/trash.rb} +0 -0
- /data/lib/kward/{ansi.rb → terminal/ansi.rb} +0 -0
- /data/lib/kward/{terminal_text.rb → terminal/text.rb} +0 -0
- /data/lib/kward/{transcript_export.rb → transcripts/transcript_export.rb} +0 -0
- /data/lib/kward/{project_files.rb → workspace/files.rb} +0 -0
- /data/lib/kward/{path_guard.rb → workspace/path_guard.rb} +0 -0
|
@@ -0,0 +1,946 @@
|
|
|
1
|
+
require "shellwords"
|
|
2
|
+
require_relative "native_arithmetic"
|
|
3
|
+
require_relative "native_condition"
|
|
4
|
+
|
|
5
|
+
# Namespace for the Kward CLI agent runtime.
|
|
6
|
+
module Kward
|
|
7
|
+
# The deliberately small native kwsh grammar. Words retain quoting information
|
|
8
|
+
# until execution; parsing never expands variables or launches commands.
|
|
9
|
+
module NativeShell
|
|
10
|
+
class SyntaxError < StandardError; end
|
|
11
|
+
class IncompleteInput < SyntaxError; end
|
|
12
|
+
|
|
13
|
+
Reference = Struct.new(:name, :subscript, :append, keyword_init: true)
|
|
14
|
+
Part = Struct.new(:kind, :text, :quoted, keyword_init: true)
|
|
15
|
+
Word = Struct.new(:parts, keyword_init: true) do
|
|
16
|
+
def literal
|
|
17
|
+
return unless parts.all? { |part| part.kind == :literal && !part.quoted }
|
|
18
|
+
|
|
19
|
+
parts.map(&:text).join
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def assignment
|
|
23
|
+
first = parts.first
|
|
24
|
+
return unless first && first.kind == :literal && !first.quoted
|
|
25
|
+
if (array = first.text.match(/\A([A-Za-z_][A-Za-z0-9_]*)\[/))
|
|
26
|
+
return subscript_assignment(array[0].length, array[1])
|
|
27
|
+
end
|
|
28
|
+
match = first.text.match(/\A([A-Za-z_][A-Za-z0-9_]*)=(.*)\z/m)
|
|
29
|
+
return unless match
|
|
30
|
+
|
|
31
|
+
value = [Part.new(kind: :literal, text: match[2], quoted: false)] + parts.drop(1)
|
|
32
|
+
[match[1], Word.new(parts: value)]
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def array_element
|
|
36
|
+
return unless parts.first&.kind == :literal && !parts.first.quoted && parts.first.text.start_with?("[")
|
|
37
|
+
subscript_assignment(1, nil)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def subscript_assignment(prefix, name)
|
|
41
|
+
key = []
|
|
42
|
+
parts.each_with_index do |part, index|
|
|
43
|
+
text = index.zero? ? part.text[prefix..] : part.text
|
|
44
|
+
if part.kind == :literal && !part.quoted && (ending = text.match(/\](\+?)=/))
|
|
45
|
+
finish = ending.begin(0)
|
|
46
|
+
key << Part.new(kind: :literal, text: text[0...finish], quoted: false)
|
|
47
|
+
value = [Part.new(kind: :literal, text: text[ending.end(0)..], quoted: false)] + parts.drop(index + 1)
|
|
48
|
+
return [Reference.new(name: name, subscript: Word.new(parts: key), append: ending[1] == "+"), Word.new(parts: value)]
|
|
49
|
+
end
|
|
50
|
+
key << Part.new(kind: part.kind, text: text, quoted: part.quoted)
|
|
51
|
+
end
|
|
52
|
+
nil
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
Parameter = Struct.new(:name, :operator, :word, keyword_init: true)
|
|
56
|
+
HereDocument = Struct.new(:delimiter, :strip_tabs, :expand, :word, keyword_init: true)
|
|
57
|
+
Token = Struct.new(:value, :line, :source, :start, :finish, :alias_names, :alias_next, keyword_init: true)
|
|
58
|
+
Node = Struct.new(:kind, :values, :source, :line, keyword_init: true)
|
|
59
|
+
|
|
60
|
+
class Lexer
|
|
61
|
+
OPERATORS = %w[&& || <<- << >> <> >| >& <& ;; ;& ; & | > < ( ) { }].freeze
|
|
62
|
+
|
|
63
|
+
def initialize(source, depth: 0, aliases: {}, line: 1, eof: true)
|
|
64
|
+
@source = source.dup
|
|
65
|
+
@eof = eof
|
|
66
|
+
@alias_regions = []
|
|
67
|
+
@token_end = 0
|
|
68
|
+
@aliases = aliases
|
|
69
|
+
@substitution_depth = depth
|
|
70
|
+
@index = 0
|
|
71
|
+
@line = line
|
|
72
|
+
@arithmetic_depth = 0
|
|
73
|
+
@parameter_depth = 0
|
|
74
|
+
@heredocs = []
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def tokens
|
|
78
|
+
result = []
|
|
79
|
+
while (token = next_token)
|
|
80
|
+
result << token
|
|
81
|
+
end
|
|
82
|
+
result
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def next_token
|
|
86
|
+
until current.nil?
|
|
87
|
+
line = @line
|
|
88
|
+
start = @index
|
|
89
|
+
case current
|
|
90
|
+
when " ", "\t", "\r"
|
|
91
|
+
advance
|
|
92
|
+
when "#"
|
|
93
|
+
advance until current.nil? || current == "\n"
|
|
94
|
+
when "\n"
|
|
95
|
+
advance
|
|
96
|
+
consume_heredocs
|
|
97
|
+
return token(";", line: line, start: start)
|
|
98
|
+
else
|
|
99
|
+
if @source[@index, 2] == "\\\n"
|
|
100
|
+
advance(2)
|
|
101
|
+
raise IncompleteInput, "continued input line" if current.nil? && !@eof
|
|
102
|
+
next
|
|
103
|
+
end
|
|
104
|
+
operator = OPERATORS.find { |candidate| @source[@index, candidate.length] == candidate }
|
|
105
|
+
operator = nil if ["{", "}"].include?(operator) && @source[@index + 1]&.match?(/[^\s;|&()<>]/)
|
|
106
|
+
value = if @source[@index, 2] == "(("
|
|
107
|
+
[:arithmetic, arithmetic_source]
|
|
108
|
+
elsif ["<(", ">("].include?(@source[@index, 2])
|
|
109
|
+
word
|
|
110
|
+
elsif (match = @source[@index..].match(/\A([0-9]+)(<<-|<<|>>|<>|>\||>&|<&|>|<)/))
|
|
111
|
+
advance(match[0].length)
|
|
112
|
+
[:redirect, match[1].to_i, match[2]]
|
|
113
|
+
elsif operator
|
|
114
|
+
advance(operator.length)
|
|
115
|
+
operator
|
|
116
|
+
else
|
|
117
|
+
word
|
|
118
|
+
end
|
|
119
|
+
return token(value, line: line, start: start, source: @source[start...@index])
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
raise IncompleteInput, "unclosed here-document" unless @heredocs.empty?
|
|
123
|
+
nil
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def regex_token
|
|
127
|
+
advance while current&.match?(/\s/)
|
|
128
|
+
return unless current
|
|
129
|
+
line, start = @line, @index
|
|
130
|
+
value = word(regex: true)
|
|
131
|
+
token(value, line: line, start: start, source: @source[start...@index])
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Reinsert alias text into the same lexical stream. Quotes and here-docs
|
|
135
|
+
# can cross the replacement boundary, unlike independently lexed tokens.
|
|
136
|
+
def replace_alias(token, source, names)
|
|
137
|
+
first, last = token.start, token.finish
|
|
138
|
+
delta = source.length - (last - first)
|
|
139
|
+
@alias_regions.each do |region|
|
|
140
|
+
region[:start] += delta if region[:start] >= last
|
|
141
|
+
region[:finish] += delta if region[:finish] >= last
|
|
142
|
+
end
|
|
143
|
+
@source[first...last] = source
|
|
144
|
+
@alias_regions << { start: first, finish: first + source.length, names: names,
|
|
145
|
+
trailing: source.end_with?(" ", "\t") }
|
|
146
|
+
@index = @token_end = first
|
|
147
|
+
@line = token.line
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def token(value, line:, start:, source: nil)
|
|
151
|
+
names = @alias_regions.select { |region| region[:start] <= start && start < region[:finish] }.flat_map { |region| region[:names] }.uniq
|
|
152
|
+
following = @alias_regions.any? { |region| region[:trailing] && @token_end < region[:finish] && region[:finish] <= start }
|
|
153
|
+
@token_end = @index
|
|
154
|
+
Token.new(value: value, line: line, source: source, start: start, finish: @index,
|
|
155
|
+
alias_names: names, alias_next: following)
|
|
156
|
+
end
|
|
157
|
+
private :token
|
|
158
|
+
|
|
159
|
+
def queue_heredoc(document)
|
|
160
|
+
@heredocs << document
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def heredoc_word
|
|
164
|
+
parts = []
|
|
165
|
+
until current.nil?
|
|
166
|
+
case current
|
|
167
|
+
when "$" then parameter(parts, true)
|
|
168
|
+
when "`" then backticks(parts, true)
|
|
169
|
+
when "\\"
|
|
170
|
+
advance
|
|
171
|
+
if ['$', '`', "\\", "\n"].include?(current)
|
|
172
|
+
literal(parts, current, true) unless current == "\n"
|
|
173
|
+
advance
|
|
174
|
+
else
|
|
175
|
+
literal(parts, "\\", true)
|
|
176
|
+
end
|
|
177
|
+
else
|
|
178
|
+
literal(parts, current, true)
|
|
179
|
+
advance
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
Word.new(parts: parts)
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
private
|
|
186
|
+
|
|
187
|
+
def consume_heredocs
|
|
188
|
+
pending = @heredocs
|
|
189
|
+
@heredocs = []
|
|
190
|
+
pending.each do |document|
|
|
191
|
+
body = +""
|
|
192
|
+
loop do
|
|
193
|
+
line = +""
|
|
194
|
+
finish = nil
|
|
195
|
+
loop do
|
|
196
|
+
raise IncompleteInput, "unclosed here-document: #{document.delimiter}" unless current
|
|
197
|
+
finish = @source.index("\n", @index)
|
|
198
|
+
fragment = @source[@index...(finish || @source.length)]
|
|
199
|
+
advance(fragment.length + (finish ? 1 : 0))
|
|
200
|
+
line << fragment
|
|
201
|
+
break unless document.expand && finish && line[/\\+\z/].to_s.length.odd?
|
|
202
|
+
line.chop!
|
|
203
|
+
end
|
|
204
|
+
line = line.sub(/\A\t+/, "") if document.strip_tabs
|
|
205
|
+
break if line == document.delimiter
|
|
206
|
+
body << line
|
|
207
|
+
body << "\n" if finish
|
|
208
|
+
end
|
|
209
|
+
document.word = if document.expand
|
|
210
|
+
Lexer.new(body, depth: @substitution_depth, aliases: @aliases).heredoc_word
|
|
211
|
+
else
|
|
212
|
+
Word.new(parts: [Part.new(kind: :literal, text: body, quoted: true)])
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def current
|
|
218
|
+
@source[@index]
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def advance(count = 1)
|
|
222
|
+
@line += @source[@index, count].count("\n")
|
|
223
|
+
@index += count
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
def literal(parts, text, quoted)
|
|
227
|
+
parts << Part.new(kind: :literal, text: text, quoted: quoted)
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
def word(parameter_word: false, quoted: false, stop: "}", regex: false)
|
|
231
|
+
parts = []
|
|
232
|
+
until current.nil? || (parameter_word ? current == stop : (regex ? current.match?(/\s/) : word_end?))
|
|
233
|
+
case current
|
|
234
|
+
when "'"
|
|
235
|
+
if quoted
|
|
236
|
+
literal(parts, current, true)
|
|
237
|
+
advance
|
|
238
|
+
next
|
|
239
|
+
end
|
|
240
|
+
advance
|
|
241
|
+
start = @index
|
|
242
|
+
advance until current.nil? || current == "'"
|
|
243
|
+
raise IncompleteInput, "line #{@line}: unclosed single quote" unless current
|
|
244
|
+
literal(parts, @source[start...@index], true)
|
|
245
|
+
advance
|
|
246
|
+
when '"'
|
|
247
|
+
double_quote(parts)
|
|
248
|
+
when "\\"
|
|
249
|
+
advance
|
|
250
|
+
raise IncompleteInput, "line #{@line}: trailing backslash" unless current
|
|
251
|
+
literal(parts, "\\", true) if quoted && !['$', '`', '"', "\\", "\n", "}"].include?(current)
|
|
252
|
+
continued = current == "\n"
|
|
253
|
+
literal(parts, current, true) unless continued
|
|
254
|
+
advance
|
|
255
|
+
raise IncompleteInput, "continued input line" if continued && current.nil? && !@eof
|
|
256
|
+
when "$"
|
|
257
|
+
parameter(parts, quoted)
|
|
258
|
+
when "`"
|
|
259
|
+
backticks(parts, quoted)
|
|
260
|
+
when "<", ">"
|
|
261
|
+
if !regex && ["<(", ">("].include?(@source[@index, 2])
|
|
262
|
+
direction = current
|
|
263
|
+
advance(2)
|
|
264
|
+
parts << Part.new(kind: :process, text: [direction, nested_program], quoted: false)
|
|
265
|
+
else
|
|
266
|
+
literal(parts, current, quoted)
|
|
267
|
+
advance
|
|
268
|
+
end
|
|
269
|
+
else
|
|
270
|
+
literal(parts, current, quoted)
|
|
271
|
+
advance
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
# Coalesce adjacent literals, preserving quote boundaries.
|
|
275
|
+
merged = []
|
|
276
|
+
parts.each do |part|
|
|
277
|
+
previous = merged.last
|
|
278
|
+
if previous && previous.kind == :literal && part.kind == :literal && previous.quoted == part.quoted
|
|
279
|
+
previous.text += part.text
|
|
280
|
+
else
|
|
281
|
+
merged << part
|
|
282
|
+
end
|
|
283
|
+
end
|
|
284
|
+
Word.new(parts: merged)
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
def double_quote(parts)
|
|
288
|
+
advance
|
|
289
|
+
start = parts.length
|
|
290
|
+
until current.nil? || current == '"'
|
|
291
|
+
case current
|
|
292
|
+
when "$"
|
|
293
|
+
parameter(parts, true)
|
|
294
|
+
when "`"
|
|
295
|
+
backticks(parts, true)
|
|
296
|
+
when "\\"
|
|
297
|
+
advance
|
|
298
|
+
raise IncompleteInput, "line #{@line}: unclosed double quote" unless current
|
|
299
|
+
if ['$', '`', '"', "\\", "\n"].include?(current)
|
|
300
|
+
literal(parts, current, true) unless current == "\n"
|
|
301
|
+
else
|
|
302
|
+
literal(parts, "\\#{current}", true)
|
|
303
|
+
end
|
|
304
|
+
advance
|
|
305
|
+
else
|
|
306
|
+
literal(parts, current, true)
|
|
307
|
+
advance
|
|
308
|
+
end
|
|
309
|
+
end
|
|
310
|
+
raise IncompleteInput, "line #{@line}: unclosed double quote" unless current
|
|
311
|
+
literal(parts, "", true) if parts.length == start
|
|
312
|
+
advance
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
def arithmetic(parts, quoted)
|
|
316
|
+
@arithmetic_depth += 1
|
|
317
|
+
raise SyntaxError, "arithmetic nesting exceeds 100 levels" if @arithmetic_depth > 100
|
|
318
|
+
advance(2)
|
|
319
|
+
depth = 0
|
|
320
|
+
expression = []
|
|
321
|
+
until current.nil?
|
|
322
|
+
if current == ")" && depth.zero?
|
|
323
|
+
raise IncompleteInput, "line #{@line}: unclosed arithmetic expansion; expected ))" unless @source[@index + 1]
|
|
324
|
+
closing = @source[@index..][/\A\)[ \t]*\)/]
|
|
325
|
+
raise SyntaxError, "line #{@line}: expected a second ) to close arithmetic expansion" unless closing
|
|
326
|
+
word = Word.new(parts: expression)
|
|
327
|
+
Arithmetic::Parser.new(word.literal.to_s).parse if word.literal
|
|
328
|
+
advance(closing.length)
|
|
329
|
+
parts << Part.new(kind: :arithmetic, text: word, quoted: quoted)
|
|
330
|
+
return
|
|
331
|
+
end
|
|
332
|
+
case current
|
|
333
|
+
when "$" then parameter(expression, false)
|
|
334
|
+
when "`" then backticks(expression, false)
|
|
335
|
+
when "\\"
|
|
336
|
+
advance
|
|
337
|
+
raise IncompleteInput, "unclosed arithmetic expansion" unless current
|
|
338
|
+
literal(expression, current, false) unless current == "\n"
|
|
339
|
+
advance
|
|
340
|
+
else
|
|
341
|
+
depth += 1 if current == "("
|
|
342
|
+
depth -= 1 if current == ")"
|
|
343
|
+
literal(expression, current, false)
|
|
344
|
+
advance
|
|
345
|
+
end
|
|
346
|
+
end
|
|
347
|
+
raise IncompleteInput, "line #{@line}: unclosed arithmetic expansion; expected ))"
|
|
348
|
+
rescue Arithmetic::Error => e
|
|
349
|
+
raise SyntaxError, "line #{@line}: #{e.message}"
|
|
350
|
+
ensure
|
|
351
|
+
@arithmetic_depth -= 1
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
def arithmetic_source
|
|
355
|
+
advance(2)
|
|
356
|
+
start = @index
|
|
357
|
+
depth = 0
|
|
358
|
+
until current.nil?
|
|
359
|
+
if current == "("
|
|
360
|
+
depth += 1
|
|
361
|
+
elsif current == ")"
|
|
362
|
+
if depth.zero?
|
|
363
|
+
raise IncompleteInput, "line #{@line}: unclosed arithmetic expansion; expected ))" unless @source[@index + 1]
|
|
364
|
+
# kwsh permits horizontal whitespace between the final closers.
|
|
365
|
+
# Recognize it here, never by rewriting quoted text or commands.
|
|
366
|
+
closing = @source[@index..][/\A\)[ \t]*\)/]
|
|
367
|
+
unless closing
|
|
368
|
+
raise SyntaxError, "line #{@line}: expected a second ) to close arithmetic expansion"
|
|
369
|
+
end
|
|
370
|
+
expression = @source[start...@index]
|
|
371
|
+
advance(closing.length)
|
|
372
|
+
return expression
|
|
373
|
+
end
|
|
374
|
+
depth -= 1
|
|
375
|
+
end
|
|
376
|
+
advance
|
|
377
|
+
end
|
|
378
|
+
raise IncompleteInput, "line #{@line}: unclosed arithmetic expansion; expected ))"
|
|
379
|
+
rescue Arithmetic::Error => e
|
|
380
|
+
raise SyntaxError, "line #{@line}: #{e.message}"
|
|
381
|
+
end
|
|
382
|
+
|
|
383
|
+
def parameter(parts, quoted)
|
|
384
|
+
advance
|
|
385
|
+
if @source[@index, 2] == "(("
|
|
386
|
+
arithmetic(parts, quoted)
|
|
387
|
+
return
|
|
388
|
+
end
|
|
389
|
+
if current == "("
|
|
390
|
+
advance
|
|
391
|
+
parts << Part.new(kind: :command, text: nested_program, quoted: quoted)
|
|
392
|
+
return
|
|
393
|
+
end
|
|
394
|
+
if current == "'" && !quoted
|
|
395
|
+
dollar_quote(parts)
|
|
396
|
+
return
|
|
397
|
+
end
|
|
398
|
+
if current == "{"
|
|
399
|
+
braced_parameter(parts, quoted)
|
|
400
|
+
return
|
|
401
|
+
elsif (match = @source[@index..].to_s.match(/\A(?:[A-Za-z_][A-Za-z0-9_]*|[0-9?@#*$!-])/))
|
|
402
|
+
name = match[0]
|
|
403
|
+
advance(name.length)
|
|
404
|
+
elsif ["!", "-"].include?(current)
|
|
405
|
+
raise SyntaxError, "line #{@line}: unsupported special parameter $#{current}"
|
|
406
|
+
else
|
|
407
|
+
literal(parts, "$", quoted)
|
|
408
|
+
return
|
|
409
|
+
end
|
|
410
|
+
parts << Part.new(kind: :parameter, text: name, quoted: quoted)
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
def dollar_quote(parts)
|
|
414
|
+
advance
|
|
415
|
+
value = +"".b
|
|
416
|
+
escapes = { "a" => "\a", "b" => "\b", "e" => "\e", "E" => "\e", "f" => "\f", "n" => "\n", "r" => "\r", "t" => "\t", "v" => "\v" }
|
|
417
|
+
until current.nil? || current == "'"
|
|
418
|
+
if current != "\\"
|
|
419
|
+
value << current.b
|
|
420
|
+
advance
|
|
421
|
+
next
|
|
422
|
+
end
|
|
423
|
+
advance
|
|
424
|
+
raise IncompleteInput, "unclosed dollar-single-quote" unless current
|
|
425
|
+
escape = current
|
|
426
|
+
advance
|
|
427
|
+
if escapes.key?(escape)
|
|
428
|
+
value << escapes.fetch(escape)
|
|
429
|
+
elsif ["\\", "'", '"'].include?(escape)
|
|
430
|
+
value << escape
|
|
431
|
+
elsif escape == "c" && current
|
|
432
|
+
value << (current.upcase.ord & 31).chr
|
|
433
|
+
advance(current == "\\" && @source[@index + 1] == "\\" ? 2 : 1)
|
|
434
|
+
elsif %w[x u U].include?(escape)
|
|
435
|
+
maximum = { "x" => 2, "u" => 4, "U" => 8 }.fetch(escape)
|
|
436
|
+
digits = @source[@index..][/\A[0-9a-fA-F]{1,#{maximum}}/]
|
|
437
|
+
if digits
|
|
438
|
+
value << digits.to_i(16).chr(escape == "x" ? Encoding::BINARY : Encoding::UTF_8).b
|
|
439
|
+
advance(digits.length)
|
|
440
|
+
else
|
|
441
|
+
value << "\\#{escape}"
|
|
442
|
+
end
|
|
443
|
+
elsif escape.match?(/[0-7]/)
|
|
444
|
+
extra = @source[@index..][/\A[0-7]{0,2}/]
|
|
445
|
+
value << ((escape + extra).to_i(8) & 255).chr(Encoding::BINARY)
|
|
446
|
+
advance(extra.length)
|
|
447
|
+
else
|
|
448
|
+
value << "\\#{escape}"
|
|
449
|
+
end
|
|
450
|
+
end
|
|
451
|
+
raise IncompleteInput, "unclosed dollar-single-quote" unless current
|
|
452
|
+
advance
|
|
453
|
+
literal(parts, value.split("\0", 2).first.to_s.force_encoding(Encoding.default_external), true)
|
|
454
|
+
rescue RangeError
|
|
455
|
+
raise SyntaxError, "invalid character escape"
|
|
456
|
+
end
|
|
457
|
+
|
|
458
|
+
def word_end?
|
|
459
|
+
current.match?(/[\s;&|<>()]/) && !["<(", ">("].include?(@source[@index, 2])
|
|
460
|
+
end
|
|
461
|
+
|
|
462
|
+
def nested_program
|
|
463
|
+
@substitution_depth += 1
|
|
464
|
+
raise SyntaxError, "substitution nesting exceeds 100 levels" if @substitution_depth > 100
|
|
465
|
+
previous_heredocs = @heredocs
|
|
466
|
+
@heredocs = []
|
|
467
|
+
Parser.new(nil, lexer: self, aliases: @aliases).parse_until_close
|
|
468
|
+
ensure
|
|
469
|
+
@heredocs = previous_heredocs if previous_heredocs
|
|
470
|
+
@substitution_depth -= 1
|
|
471
|
+
end
|
|
472
|
+
|
|
473
|
+
def backticks(parts, quoted)
|
|
474
|
+
advance
|
|
475
|
+
source = +""
|
|
476
|
+
until current.nil? || current == "`"
|
|
477
|
+
if current == "\\" && ['$', '`', "\\", "\n"].include?(@source[@index + 1])
|
|
478
|
+
advance
|
|
479
|
+
source << current unless current == "\n"
|
|
480
|
+
else
|
|
481
|
+
source << current
|
|
482
|
+
end
|
|
483
|
+
advance
|
|
484
|
+
end
|
|
485
|
+
raise IncompleteInput, "line #{@line}: unclosed command substitution" unless current
|
|
486
|
+
advance
|
|
487
|
+
raise SyntaxError, "substitution nesting exceeds 100 levels" if @substitution_depth >= 100
|
|
488
|
+
lexer = Lexer.new(source, depth: @substitution_depth + 1, aliases: @aliases)
|
|
489
|
+
parts << Part.new(kind: :command, text: Parser.new(nil, lexer: lexer, aliases: @aliases).parse, quoted: quoted)
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
def braced_parameter(parts, quoted)
|
|
493
|
+
@parameter_depth += 1
|
|
494
|
+
raise SyntaxError, "parameter nesting exceeds 100 levels" if @parameter_depth > 100
|
|
495
|
+
advance
|
|
496
|
+
length = current == "#" && @source[@index + 1] != "}"
|
|
497
|
+
keys = current == "!" && @source[@index + 1] != "}"
|
|
498
|
+
advance if length || keys
|
|
499
|
+
name = @source[@index..][/\A(?:[A-Za-z_][A-Za-z0-9_]*|[0-9]+|[?@#*$!-])/]
|
|
500
|
+
raise IncompleteInput, "line #{@line}: unclosed parameter expansion" unless current
|
|
501
|
+
raise SyntaxError, "line #{@line}: unsupported parameter expansion" unless name
|
|
502
|
+
advance(name.length)
|
|
503
|
+
if current == "["
|
|
504
|
+
advance
|
|
505
|
+
subscript = word(parameter_word: true, stop: "]")
|
|
506
|
+
raise IncompleteInput, "unclosed array subscript" unless current == "]"
|
|
507
|
+
advance
|
|
508
|
+
name = Reference.new(name: name, subscript: subscript)
|
|
509
|
+
end
|
|
510
|
+
if keys && (!name.is_a?(Reference) || !%w[@ *].include?(name.subscript.literal))
|
|
511
|
+
raise SyntaxError, "indirection requires an array @ or * reference"
|
|
512
|
+
end
|
|
513
|
+
if current == "}"
|
|
514
|
+
advance
|
|
515
|
+
value = length || keys ? Parameter.new(name: name, operator: keys ? :keys : :length) : name
|
|
516
|
+
else
|
|
517
|
+
operator = @source[@index..][/\A(?::[-=+?]|[-=+?]|##?|%%?)/] unless length || keys
|
|
518
|
+
raise IncompleteInput, "line #{@line}: unclosed parameter expansion" unless current
|
|
519
|
+
raise SyntaxError, "line #{@line}: unsupported parameter expansion" unless operator
|
|
520
|
+
advance(operator.length)
|
|
521
|
+
pattern = operator.start_with?("#", "%")
|
|
522
|
+
operand = word(parameter_word: true, quoted: quoted && !pattern)
|
|
523
|
+
raise IncompleteInput, "line #{@line}: unclosed parameter expansion" unless current == "}"
|
|
524
|
+
advance
|
|
525
|
+
value = Parameter.new(name: name, operator: operator, word: operand)
|
|
526
|
+
end
|
|
527
|
+
parts << Part.new(kind: :parameter, text: value, quoted: quoted)
|
|
528
|
+
ensure
|
|
529
|
+
@parameter_depth -= 1
|
|
530
|
+
end
|
|
531
|
+
end
|
|
532
|
+
|
|
533
|
+
class Parser
|
|
534
|
+
RESERVED = %w[if then elif else fi for in do done while until case esac select function time !].freeze
|
|
535
|
+
REDIRECTS = %w[< > >> <> >| >& <& << <<-].freeze
|
|
536
|
+
|
|
537
|
+
def self.parse(source, aliases: {}, line: 1, eof: true)
|
|
538
|
+
new(source, aliases: aliases, line: line, eof: eof).parse
|
|
539
|
+
end
|
|
540
|
+
|
|
541
|
+
def initialize(source, lexer: nil, aliases: {}, line: 1, eof: true)
|
|
542
|
+
raise SyntaxError, "NUL bytes are not allowed in shell source" if source&.include?("\0")
|
|
543
|
+
@lexer = lexer || Lexer.new(source, aliases: aliases, line: line, eof: eof)
|
|
544
|
+
@aliases = aliases
|
|
545
|
+
@alias_tokens = 0
|
|
546
|
+
@alias_bytes = 0
|
|
547
|
+
@tokens = []
|
|
548
|
+
@index = 0
|
|
549
|
+
@depth = 0
|
|
550
|
+
end
|
|
551
|
+
|
|
552
|
+
def parse
|
|
553
|
+
tree = sequence
|
|
554
|
+
error("unexpected token") if peek
|
|
555
|
+
tree
|
|
556
|
+
end
|
|
557
|
+
|
|
558
|
+
def parse_until_close
|
|
559
|
+
tree = sequence([")"])
|
|
560
|
+
expect(")")
|
|
561
|
+
tree
|
|
562
|
+
end
|
|
563
|
+
|
|
564
|
+
private
|
|
565
|
+
|
|
566
|
+
def node(kind, *values)
|
|
567
|
+
Node.new(kind: kind, values: values)
|
|
568
|
+
end
|
|
569
|
+
|
|
570
|
+
def peek(offset = 0)
|
|
571
|
+
while @tokens.length <= @index + offset
|
|
572
|
+
token = @lexer.next_token
|
|
573
|
+
break unless token
|
|
574
|
+
@alias_tokens += 1 unless token.alias_names.to_a.empty?
|
|
575
|
+
raise SyntaxError, "alias expansion exceeds 65536 tokens" if @alias_tokens > 65_536
|
|
576
|
+
@tokens << token
|
|
577
|
+
end
|
|
578
|
+
@tokens[@index + offset]&.value
|
|
579
|
+
end
|
|
580
|
+
|
|
581
|
+
def text
|
|
582
|
+
peek.is_a?(Word) ? peek.literal : peek
|
|
583
|
+
end
|
|
584
|
+
|
|
585
|
+
def take
|
|
586
|
+
token = peek
|
|
587
|
+
@index += 1
|
|
588
|
+
token
|
|
589
|
+
end
|
|
590
|
+
|
|
591
|
+
def accept(value)
|
|
592
|
+
return false unless text == value
|
|
593
|
+
take
|
|
594
|
+
true
|
|
595
|
+
end
|
|
596
|
+
|
|
597
|
+
def expect(value)
|
|
598
|
+
error("expected #{value.inspect}") unless accept(value)
|
|
599
|
+
end
|
|
600
|
+
|
|
601
|
+
def error(message)
|
|
602
|
+
token = @tokens[@index]
|
|
603
|
+
klass = token ? SyntaxError : IncompleteInput
|
|
604
|
+
raise klass, "line #{token&.line || @tokens.last&.line || 1}: #{message}"
|
|
605
|
+
end
|
|
606
|
+
|
|
607
|
+
def separators
|
|
608
|
+
take while text == ";"
|
|
609
|
+
end
|
|
610
|
+
|
|
611
|
+
def sequence(stops = [])
|
|
612
|
+
commands = []
|
|
613
|
+
separators
|
|
614
|
+
until peek.nil? || stops.include?(text)
|
|
615
|
+
expand_alias
|
|
616
|
+
break if peek.nil? || stops.include?(text)
|
|
617
|
+
if text == ";" && @tokens[@index].source.nil?
|
|
618
|
+
separators
|
|
619
|
+
next
|
|
620
|
+
end
|
|
621
|
+
command = and_or
|
|
622
|
+
background = accept("&")
|
|
623
|
+
commands << (background ? node(:background, command) : command)
|
|
624
|
+
break if peek.nil? || [";;", ";&", ")"].include?(text) && stops.include?(text)
|
|
625
|
+
error("expected command separator") unless background || accept(";")
|
|
626
|
+
separators
|
|
627
|
+
end
|
|
628
|
+
node(:sequence, commands)
|
|
629
|
+
end
|
|
630
|
+
|
|
631
|
+
def and_or
|
|
632
|
+
first = pipeline
|
|
633
|
+
rest = []
|
|
634
|
+
while ["&&", "||"].include?(text)
|
|
635
|
+
operator = take
|
|
636
|
+
separators
|
|
637
|
+
rest << [operator, pipeline]
|
|
638
|
+
end
|
|
639
|
+
rest.empty? ? first : node(:and_or, first, rest)
|
|
640
|
+
end
|
|
641
|
+
|
|
642
|
+
def expand_alias
|
|
643
|
+
loop do
|
|
644
|
+
name = peek.is_a?(Word) && peek.literal
|
|
645
|
+
break unless name && @aliases.key?(name) && !RESERVED.include?(name)
|
|
646
|
+
token = @tokens[@index]
|
|
647
|
+
names = token.alias_names || []
|
|
648
|
+
break if names.include?(name)
|
|
649
|
+
raise SyntaxError, "alias expansion exceeds 100 levels" if names.length >= 100
|
|
650
|
+
replacement = @aliases.fetch(name)
|
|
651
|
+
@alias_bytes += replacement.bytesize
|
|
652
|
+
raise SyntaxError, "alias replacement exceeds 1048576 bytes" if @alias_bytes > 1_048_576
|
|
653
|
+
@lexer.replace_alias(token, replacement, names + [name])
|
|
654
|
+
@tokens.slice!(@index..)
|
|
655
|
+
end
|
|
656
|
+
end
|
|
657
|
+
|
|
658
|
+
def pipeline
|
|
659
|
+
start = @index
|
|
660
|
+
expand_alias
|
|
661
|
+
negate = accept("!")
|
|
662
|
+
commands = [command]
|
|
663
|
+
while accept("|")
|
|
664
|
+
separators
|
|
665
|
+
commands << command
|
|
666
|
+
end
|
|
667
|
+
result = node(:pipeline, commands, negate)
|
|
668
|
+
result.source = @tokens[start...@index].filter_map(&:source).join(" ")
|
|
669
|
+
result
|
|
670
|
+
end
|
|
671
|
+
|
|
672
|
+
def command
|
|
673
|
+
expand_alias
|
|
674
|
+
peek
|
|
675
|
+
line = @tokens[@index]&.line
|
|
676
|
+
@depth += 1
|
|
677
|
+
raise SyntaxError, "shell syntax nesting exceeds 100 levels" if @depth > 100
|
|
678
|
+
result = if peek.is_a?(Array) && peek.first == :arithmetic
|
|
679
|
+
expression = take.last
|
|
680
|
+
validate_arithmetic(expression)
|
|
681
|
+
node(:arithmetic, expression)
|
|
682
|
+
else
|
|
683
|
+
compound_or_simple
|
|
684
|
+
end
|
|
685
|
+
if result.kind != :simple
|
|
686
|
+
redirects = []
|
|
687
|
+
redirects << redirect while redirection?
|
|
688
|
+
result = node(:redirect, result, redirects) unless redirects.empty?
|
|
689
|
+
end
|
|
690
|
+
result.line = line
|
|
691
|
+
result
|
|
692
|
+
ensure
|
|
693
|
+
@depth -= 1
|
|
694
|
+
end
|
|
695
|
+
|
|
696
|
+
def validate_arithmetic(expression)
|
|
697
|
+
Arithmetic::Parser.new(expression).parse unless expression.strip.empty?
|
|
698
|
+
rescue Arithmetic::Error => e
|
|
699
|
+
error(e.message)
|
|
700
|
+
end
|
|
701
|
+
|
|
702
|
+
def compound_or_simple
|
|
703
|
+
case text
|
|
704
|
+
when "if" then conditional
|
|
705
|
+
when "for" then for_loop
|
|
706
|
+
when "case" then case_command
|
|
707
|
+
when "[["
|
|
708
|
+
take
|
|
709
|
+
condition = test_expression
|
|
710
|
+
expect("]]")
|
|
711
|
+
node(:test, condition)
|
|
712
|
+
when "while", "until" then condition_loop
|
|
713
|
+
when "{" then group
|
|
714
|
+
when "("
|
|
715
|
+
take
|
|
716
|
+
node(:subshell, parse_until_close)
|
|
717
|
+
else
|
|
718
|
+
append = peek.is_a?(Word) && peek.literal&.match(/\A([A-Za-z_][A-Za-z0-9_]*)\+=\z/)
|
|
719
|
+
if peek.is_a?(Word) && (peek.assignment || append) && peek(1) == "("
|
|
720
|
+
word = take
|
|
721
|
+
name, value = append ? [append[1], Word.new(parts: [])] : word.assignment
|
|
722
|
+
error("invalid array assignment") unless name.is_a?(String) && value.parts.all? { |part| part.kind == :literal && part.text.empty? }
|
|
723
|
+
node(:array_assign, name, array_elements, !!append)
|
|
724
|
+
elsif peek.is_a?(Word) && peek(1) == "("
|
|
725
|
+
function
|
|
726
|
+
else
|
|
727
|
+
simple
|
|
728
|
+
end
|
|
729
|
+
end
|
|
730
|
+
end
|
|
731
|
+
|
|
732
|
+
def array_elements
|
|
733
|
+
expect("(")
|
|
734
|
+
elements = []
|
|
735
|
+
until text == ")"
|
|
736
|
+
next if accept(";")
|
|
737
|
+
error("expected array element") unless peek.is_a?(Word)
|
|
738
|
+
elements << take
|
|
739
|
+
end
|
|
740
|
+
expect(")")
|
|
741
|
+
elements
|
|
742
|
+
end
|
|
743
|
+
|
|
744
|
+
def test_expression(minimum = 0)
|
|
745
|
+
@depth += 1
|
|
746
|
+
raise SyntaxError, "conditional nesting exceeds 100 levels" if @depth > 100
|
|
747
|
+
left = if accept("!")
|
|
748
|
+
[:not, test_expression(3)]
|
|
749
|
+
elsif accept("(")
|
|
750
|
+
result = test_expression
|
|
751
|
+
expect(")")
|
|
752
|
+
result
|
|
753
|
+
elsif Condition::UNARY.include?(text)
|
|
754
|
+
operator = take.literal
|
|
755
|
+
[:unary, operator, test_word]
|
|
756
|
+
else
|
|
757
|
+
word = test_word
|
|
758
|
+
if Condition::BINARY.include?(text)
|
|
759
|
+
operator = text
|
|
760
|
+
take
|
|
761
|
+
[:binary, operator, word, test_word(regex: operator == "=~")]
|
|
762
|
+
else
|
|
763
|
+
[:string, word]
|
|
764
|
+
end
|
|
765
|
+
end
|
|
766
|
+
while %w[&& ||].include?(text) && (text == "&&" ? 2 : 1) >= minimum
|
|
767
|
+
operator = take
|
|
768
|
+
left = [operator == "&&" ? :and : :or, left, test_expression(operator == "&&" ? 3 : 2)]
|
|
769
|
+
end
|
|
770
|
+
left
|
|
771
|
+
ensure
|
|
772
|
+
@depth -= 1
|
|
773
|
+
end
|
|
774
|
+
|
|
775
|
+
def test_word(regex: false)
|
|
776
|
+
if regex && @tokens.length == @index
|
|
777
|
+
token = @lexer.regex_token
|
|
778
|
+
@tokens << token if token
|
|
779
|
+
end
|
|
780
|
+
error("expected conditional operand") unless peek.is_a?(Word) && text != "]]"
|
|
781
|
+
take
|
|
782
|
+
end
|
|
783
|
+
|
|
784
|
+
def conditional(depth = 0)
|
|
785
|
+
raise SyntaxError, "conditional nesting exceeds 100 levels" if depth >= 100
|
|
786
|
+
take
|
|
787
|
+
condition = nonempty_sequence(["then"])
|
|
788
|
+
expect("then")
|
|
789
|
+
yes = nonempty_sequence(%w[elif else fi])
|
|
790
|
+
no = if text == "elif"
|
|
791
|
+
conditional(depth + 1)
|
|
792
|
+
elsif accept("else")
|
|
793
|
+
branch = nonempty_sequence(["fi"])
|
|
794
|
+
expect("fi")
|
|
795
|
+
branch
|
|
796
|
+
else
|
|
797
|
+
expect("fi")
|
|
798
|
+
node(:sequence, [])
|
|
799
|
+
end
|
|
800
|
+
node(:if, condition, yes, no)
|
|
801
|
+
end
|
|
802
|
+
|
|
803
|
+
def nonempty_sequence(stops)
|
|
804
|
+
result = sequence(stops)
|
|
805
|
+
error("expected command") if result.values.first.empty?
|
|
806
|
+
result
|
|
807
|
+
end
|
|
808
|
+
|
|
809
|
+
def for_loop
|
|
810
|
+
take
|
|
811
|
+
if peek.is_a?(Array) && peek.first == :arithmetic
|
|
812
|
+
expressions = take.last.split(";", -1)
|
|
813
|
+
error("arithmetic for requires init; condition; update") unless expressions.length == 3
|
|
814
|
+
expressions.each { |expression| validate_arithmetic(expression) }
|
|
815
|
+
separators
|
|
816
|
+
expect("do")
|
|
817
|
+
body = nonempty_sequence(["done"])
|
|
818
|
+
expect("done")
|
|
819
|
+
return node(:arithmetic_for, expressions, body)
|
|
820
|
+
end
|
|
821
|
+
name = take
|
|
822
|
+
error("expected loop variable") unless name.is_a?(Word) && name.literal&.match?(/\A[A-Za-z_][A-Za-z0-9_]*\z/)
|
|
823
|
+
words = nil
|
|
824
|
+
if accept("in")
|
|
825
|
+
words = []
|
|
826
|
+
words << take while peek.is_a?(Word)
|
|
827
|
+
end
|
|
828
|
+
expect(";")
|
|
829
|
+
separators
|
|
830
|
+
expect("do")
|
|
831
|
+
body = nonempty_sequence(["done"])
|
|
832
|
+
expect("done")
|
|
833
|
+
node(:for, name.literal, words, body)
|
|
834
|
+
end
|
|
835
|
+
|
|
836
|
+
def case_command
|
|
837
|
+
take
|
|
838
|
+
subject = take
|
|
839
|
+
error("expected case word") unless subject.is_a?(Word)
|
|
840
|
+
separators
|
|
841
|
+
expect("in")
|
|
842
|
+
separators
|
|
843
|
+
clauses = []
|
|
844
|
+
until text == "esac"
|
|
845
|
+
accept("(")
|
|
846
|
+
patterns = []
|
|
847
|
+
loop do
|
|
848
|
+
pattern = take
|
|
849
|
+
error("expected case pattern") unless pattern.is_a?(Word)
|
|
850
|
+
patterns << pattern
|
|
851
|
+
break unless accept("|")
|
|
852
|
+
end
|
|
853
|
+
expect(")")
|
|
854
|
+
body = sequence(%w[;; ;& esac])
|
|
855
|
+
terminator = take if %w[;; ;&].include?(text)
|
|
856
|
+
clauses << [patterns, body, terminator == ";&"]
|
|
857
|
+
break unless terminator
|
|
858
|
+
separators
|
|
859
|
+
end
|
|
860
|
+
expect("esac")
|
|
861
|
+
node(:case, subject, clauses)
|
|
862
|
+
end
|
|
863
|
+
|
|
864
|
+
def condition_loop
|
|
865
|
+
kind = take.literal
|
|
866
|
+
condition = nonempty_sequence(["do"])
|
|
867
|
+
expect("do")
|
|
868
|
+
body = nonempty_sequence(["done"])
|
|
869
|
+
expect("done")
|
|
870
|
+
node(:loop, kind, condition, body)
|
|
871
|
+
end
|
|
872
|
+
|
|
873
|
+
def group
|
|
874
|
+
expect("{")
|
|
875
|
+
body = nonempty_sequence(["}"])
|
|
876
|
+
expect("}")
|
|
877
|
+
body
|
|
878
|
+
end
|
|
879
|
+
|
|
880
|
+
def function
|
|
881
|
+
name = take.literal
|
|
882
|
+
error("invalid function name") unless name&.match?(/\A[A-Za-z_][A-Za-z0-9_]*\z/)
|
|
883
|
+
expect("(")
|
|
884
|
+
expect(")")
|
|
885
|
+
separators
|
|
886
|
+
node(:function, name, group)
|
|
887
|
+
end
|
|
888
|
+
|
|
889
|
+
def redirection?
|
|
890
|
+
peek.is_a?(Array) && peek.first == :redirect || REDIRECTS.include?(peek)
|
|
891
|
+
end
|
|
892
|
+
|
|
893
|
+
def redirect
|
|
894
|
+
operator = take
|
|
895
|
+
if operator.is_a?(Array)
|
|
896
|
+
_, descriptor, operator = operator
|
|
897
|
+
else
|
|
898
|
+
descriptor = operator.start_with?("<") ? 0 : 1
|
|
899
|
+
end
|
|
900
|
+
error("only file descriptors 0 through 9 are supported") unless (0..9).cover?(descriptor)
|
|
901
|
+
target = take
|
|
902
|
+
error("expected redirection target") unless target.is_a?(Word)
|
|
903
|
+
if ["<<", "<<-"].include?(operator)
|
|
904
|
+
raw = @tokens[@index - 1].source
|
|
905
|
+
delimiter = Shellwords.split(raw).join
|
|
906
|
+
target = HereDocument.new(delimiter: delimiter, strip_tabs: operator == "<<-", expand: !raw.match?(/["'\\]/))
|
|
907
|
+
@lexer.queue_heredoc(target)
|
|
908
|
+
end
|
|
909
|
+
[descriptor, operator, target]
|
|
910
|
+
end
|
|
911
|
+
|
|
912
|
+
def simple
|
|
913
|
+
words = []
|
|
914
|
+
redirects = []
|
|
915
|
+
while peek.is_a?(Word) || redirection?
|
|
916
|
+
expand_alias if words.all?(&:assignment) || @tokens[@index]&.alias_next
|
|
917
|
+
break unless peek.is_a?(Word) || redirection?
|
|
918
|
+
if redirection?
|
|
919
|
+
redirects << redirect
|
|
920
|
+
else
|
|
921
|
+
error("unsupported or unexpected reserved word #{text}") if words.empty? && RESERVED.include?(text)
|
|
922
|
+
error("[[ is not supported; use test or [") if words.empty? && text == "[["
|
|
923
|
+
words << take
|
|
924
|
+
end
|
|
925
|
+
end
|
|
926
|
+
if text == "(" && words.first&.literal == "local" && words.last&.assignment
|
|
927
|
+
name, value = words.last.assignment
|
|
928
|
+
unless words.length == 2 && name.is_a?(String) && value.parts.all? { |part| part.kind == :literal && part.text.empty? }
|
|
929
|
+
error("local array declaration requires name=(...)")
|
|
930
|
+
end
|
|
931
|
+
result = node(:local_array, name, array_elements)
|
|
932
|
+
return redirects.empty? ? result : node(:redirect, result, redirects)
|
|
933
|
+
end
|
|
934
|
+
if text == "(" && words.first&.literal == "declare" && words.last&.assignment
|
|
935
|
+
error("array declaration requires -a or -A") unless words.length == 3 && %w[-a -A].include?(words[1].literal)
|
|
936
|
+
name, value = words.last.assignment
|
|
937
|
+
error("invalid array declaration") unless name.is_a?(String) && value.parts.all? { |part| part.kind == :literal && part.text.empty? }
|
|
938
|
+
result = node(:declare_array, words[1].literal, name, array_elements)
|
|
939
|
+
return redirects.empty? ? result : node(:redirect, result, redirects)
|
|
940
|
+
end
|
|
941
|
+
error("expected command") if words.empty? && redirects.empty?
|
|
942
|
+
node(:simple, words, redirects)
|
|
943
|
+
end
|
|
944
|
+
end
|
|
945
|
+
end
|
|
946
|
+
end
|