samagotchi 0.75.0 → 0.77.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +72 -1
  3. data/docs/architecture.md +11 -4
  4. data/docs/cli.md +13 -1
  5. data/docs/configuration.md +4 -3
  6. data/docs/guardrails.md +9 -0
  7. data/docs/internals/anthropic.md +3 -1
  8. data/docs/internals/bundles.md +20 -7
  9. data/docs/internals/context-telemetry.md +22 -0
  10. data/docs/internals/engine.md +15 -4
  11. data/docs/internals/prompt-caching.md +2 -1
  12. data/docs/memory.md +34 -0
  13. data/docs/plugins.md +43 -1
  14. data/docs/releasing.md +6 -2
  15. data/docs/sessions.md +3 -1
  16. data/lib/samagotchi/answer_command.rb +5 -5
  17. data/lib/samagotchi/bridge.rb +3 -3
  18. data/lib/samagotchi/bundle_command.rb +52 -39
  19. data/lib/samagotchi/bundles/codex-peer/manifest.yml +2 -2
  20. data/lib/samagotchi/bundles/codex-peer/plugin.rb +19 -2
  21. data/lib/samagotchi/bundles/core/manifest.yml +2 -2
  22. data/lib/samagotchi/bundles/dev/manifest.yml +2 -2
  23. data/lib/samagotchi/bundles/source-links/hooks/source_links.rb +12 -10
  24. data/lib/samagotchi/bundles/source-links/manifest.yml +2 -2
  25. data/lib/samagotchi/bundles/system/manifest.yml +1 -1
  26. data/lib/samagotchi/child_reports.rb +1 -1
  27. data/lib/samagotchi/desktop/macos.rb +1 -2
  28. data/lib/samagotchi/effective_model.rb +5 -0
  29. data/lib/samagotchi/engine.rb +105 -16
  30. data/lib/samagotchi/guardrails/parent_approvals.rb +4 -4
  31. data/lib/samagotchi/guardrails/parent_continue.rb +2 -2
  32. data/lib/samagotchi/kernel_loop.rb +13 -10
  33. data/lib/samagotchi/llm_context_override.rb +8 -6
  34. data/lib/samagotchi/memory_bundle/asset_installer.rb +34 -5
  35. data/lib/samagotchi/memory_bundle/builder.rb +13 -14
  36. data/lib/samagotchi/memory_bundle/installer.rb +122 -5
  37. data/lib/samagotchi/memory_bundle/manifest.rb +20 -1
  38. data/lib/samagotchi/memory_bundle/memory_files.rb +24 -6
  39. data/lib/samagotchi/memory_bundle/profile.rb +1 -1
  40. data/lib/samagotchi/memory_bundle/shipped_update.rb +2 -1
  41. data/lib/samagotchi/memory_bundle/source.rb +66 -3
  42. data/lib/samagotchi/memory_bundle/system_bundle.rb +4 -2
  43. data/lib/samagotchi/message_parts.rb +272 -0
  44. data/lib/samagotchi/model_price.rb +4 -3
  45. data/lib/samagotchi/parent_report.rb +30 -22
  46. data/lib/samagotchi/paths.rb +15 -1
  47. data/lib/samagotchi/pending_question.rb +49 -0
  48. data/lib/samagotchi/question_desk.rb +39 -32
  49. data/lib/samagotchi/relay.rb +8 -9
  50. data/lib/samagotchi/relay_verifier.rb +2 -2
  51. data/lib/samagotchi/relay_watcher.rb +2 -2
  52. data/lib/samagotchi/reply_wait.rb +4 -4
  53. data/lib/samagotchi/row_refs.rb +1 -1
  54. data/lib/samagotchi/session.rb +22 -14
  55. data/lib/samagotchi/session_commands.rb +5 -0
  56. data/lib/samagotchi/session_manager.rb +1 -2
  57. data/lib/samagotchi/session_metrics/model_segment.rb +109 -0
  58. data/lib/samagotchi/session_metrics/totals.rb +127 -0
  59. data/lib/samagotchi/session_metrics.rb +118 -83
  60. data/lib/samagotchi/terminal_ui/attach_launcher.rb +3 -2
  61. data/lib/samagotchi/terminal_ui/attached_loop.rb +2 -2
  62. data/lib/samagotchi/terminal_ui/formatting.rb +24 -10
  63. data/lib/samagotchi/terminal_ui/model_segments_text.rb +118 -0
  64. data/lib/samagotchi/terminal_ui/status_row.rb +3 -1
  65. data/lib/samagotchi/terminal_ui.rb +5 -3
  66. data/lib/samagotchi/tool_output_cut.rb +1 -1
  67. data/lib/samagotchi/tool_response.rb +13 -13
  68. data/lib/samagotchi/tool_row_fields.rb +1 -1
  69. data/lib/samagotchi/tool_runner.rb +16 -9
  70. data/lib/samagotchi/toolkit.rb +9 -8
  71. data/lib/samagotchi/toolkit_command.rb +2 -2
  72. data/lib/samagotchi/tools/builtins.rb +1 -5
  73. data/lib/samagotchi/tools/delegate_relay.rb +7 -7
  74. data/lib/samagotchi/tools/delegate_wait.rb +1 -1
  75. data/lib/samagotchi/tools/web_fetch.rb +3 -1
  76. data/lib/samagotchi/version.rb +1 -1
  77. data/lib/samagotchi/web/app.rb +5 -2
  78. data/lib/samagotchi/web/public/app.js +21 -58
  79. data/lib/samagotchi/web/public/chains.js +1 -1
  80. data/lib/samagotchi/web/public/composer.js +9 -5
  81. data/lib/samagotchi/web/public/ctx.js +84 -11
  82. data/lib/samagotchi/web/public/index.html +32 -36
  83. data/lib/samagotchi/web/public/info_bar_model.js +2 -1
  84. data/lib/samagotchi/web/public/markdown_place.js +53 -0
  85. data/lib/samagotchi/web/public/question_cards.js +1 -1
  86. data/lib/samagotchi/web/public/session_state.js +3 -0
  87. data/lib/samagotchi/web/public/stage_slots.js +3 -3
  88. data/lib/samagotchi/web/public/stream_handlers.js +38 -3
  89. data/lib/samagotchi/web/public/timing.js +11 -2
  90. data/lib/samagotchi/web/public/toasts.js +69 -0
  91. metadata +8 -2
  92. data/lib/samagotchi/web/message_parts.rb +0 -274
@@ -93,7 +93,7 @@ module Samagotchi
93
93
  # guardrail rules or approvals: by its rule, its paths or its command
94
94
  # (the dirs as this process's environment and config.yml name them).
95
95
  def protected?(pending)
96
- facts = fetch(pending, :approval)
96
+ facts = pending.approval
97
97
  return false unless facts.is_a?(Hash)
98
98
  return true if PROTECTED_RULES.include?(fetch(facts, :rule).to_s)
99
99
  return true if fetch(facts, :source).to_s == ProtectedPaths::SOURCE
@@ -137,13 +137,13 @@ module Samagotchi
137
137
 
138
138
  # An approval: its kind says so, or it carries approval facts.
139
139
  def approval?(pending)
140
- fetch(pending, :kind).to_s == Approval::KIND || !fetch(pending, :approval).nil?
140
+ pending.kind.to_s == Approval::KIND || !pending.approval.nil?
141
141
  end
142
142
 
143
143
  # The offered scopes, in option order, or nil when they are missing,
144
144
  # empty, not strings, or leave no option after them for Deny.
145
145
  def scopes(pending)
146
- facts = fetch(pending, :approval)
146
+ facts = pending.approval
147
147
  scopes = facts.is_a?(Hash) ? fetch(facts, :scopes) : nil
148
148
  return nil unless scopes.is_a?(Array) && !scopes.empty? && scopes.all?(String)
149
149
  return nil unless scopes.size < options(pending).size
@@ -160,7 +160,7 @@ module Samagotchi
160
160
  index == options.size - 1 && options[index].to_s == Approval::DENY
161
161
  end
162
162
 
163
- def options(pending) = Array(fetch(pending, :options))
163
+ def options(pending) = Array(pending.options)
164
164
 
165
165
  def fetch(hash, key)
166
166
  return nil unless hash.is_a?(Hash)
@@ -24,12 +24,12 @@ module Samagotchi
24
24
  end
25
25
 
26
26
  # Why a parent may not give this answer, or nil.
27
- # @param pending [Hash] the pending question (symbol or string keys)
27
+ # @param pending [PendingQuestion] the pending question
28
28
  # @param selected [Array<String>] the selected labels
29
29
  # @return [Symbol, nil] :stop_only
30
30
  def refusal(pending, selected, allowed: allowed?)
31
31
  return nil if allowed
32
- return nil unless (pending[:kind] || pending["kind"]).to_s == KIND
32
+ return nil unless pending.kind.to_s == KIND
33
33
 
34
34
  Array(selected).map(&:to_s).include?(CONTINUE) ? :stop_only : nil
35
35
  end
@@ -943,19 +943,22 @@ module Samagotchi
943
943
  private :emit_llm_context
944
944
 
945
945
  # A request's re-prefilled tokens, for both loops: of the prompt the
946
- # last request on +model+ sent (this kernel's, any turn), what the
947
- # server didn't reuse from its cache this time (that prompt less
948
- # +cached+), so an LLM context edit's cache break shows, and so does
949
- # anything else that rewrote the prompt's middle or lost the cache. nil
950
- # under TokenUsage::REPREFILL_MIN_TOKENS (a cache block's rounding), for
951
- # a server that reports no cached count (or 0: a cold cache isn't told
952
- # from no cache), the first request, a model switch, and the first
953
- # request after a warm-up (#warmed_up!).
946
+ # last request on +model+ through this client (the host) sent (this
947
+ # kernel's, any turn), what the server didn't reuse from its cache this
948
+ # time (that prompt less +cached+), so an LLM context edit's cache break
949
+ # shows, and so does anything else that rewrote the prompt's middle or
950
+ # lost the cache. nil under TokenUsage::REPREFILL_MIN_TOKENS (a cache
951
+ # block's rounding), for a server that reports no cached count (or 0: a
952
+ # cold cache isn't told from no cache), the first request, a model
953
+ # switch (another bare model, or the same one on another host: the
954
+ # Engine points #client at the new host's; SessionMetrics counts that
955
+ # prefill as the switch's), and the first request after a warm-up
956
+ # (#warmed_up!).
954
957
  # @return [Integer, nil]
955
958
  def reprefilled_tokens(model, prompt:, cached:)
956
959
  previous = @reprefill_baseline
957
- @reprefill_baseline = { model: model, prompt: prompt.to_i } if prompt.to_i.positive?
958
- return nil unless previous && previous[:model] == model && cached.to_i.positive?
960
+ @reprefill_baseline = { model: model, client: @client, prompt: prompt.to_i } if prompt.to_i.positive?
961
+ return nil unless previous && previous[:model] == model && previous[:client].equal?(@client) && cached.to_i.positive?
959
962
 
960
963
  tokens = previous[:prompt] - cached.to_i
961
964
  tokens >= TokenUsage::REPREFILL_MIN_TOKENS ? tokens : nil
@@ -125,7 +125,14 @@ module Samagotchi
125
125
  raise ArgumentError, "unknown llm_context field #{field}" unless FIELDS.include?(field)
126
126
 
127
127
  text = word.to_s.strip
128
- [field, text.casecmp?(DEFAULT_WORD) ? nil : send(:"parse_#{field}", text)]
128
+ value = if text.casecmp?(DEFAULT_WORD)
129
+ nil
130
+ elsif field == :budget_tokens
131
+ LLMContextStrategy.budget_tokens(text)
132
+ else
133
+ send(:"parse_#{field}", text)
134
+ end
135
+ [field, value]
129
136
  end
130
137
  (current || new).with(**fields)
131
138
  end
@@ -152,11 +159,6 @@ module Samagotchi
152
159
  rule || raise(ArgumentError, "unknown llm_context apply #{text} (#{LLMContextStrategy::APPLIES.join(", ")})")
153
160
  end
154
161
 
155
- # "64000", "64k" or "off" (0): LLMContextStrategy.budget_tokens.
156
- # .update reaches it (and the other parse_<field>s) by its name.
157
- # @return [Integer] 0 for off
158
- def self.parse_budget_tokens(text) = LLMContextStrategy.budget_tokens(text)
159
-
160
162
  private_class_method :file_value, :file_strategy, :file_budget
161
163
  end
162
164
  end
@@ -55,6 +55,7 @@ module Samagotchi
55
55
  prune_dropped_hooks if @upgrade && @existing && !@dry_run
56
56
  verify_hook_checksums if @manifest && @strict
57
57
  incoming_rules = Dir.glob(File.join(@bundle_dir, "guardrails", "*.yml")).sort
58
+ .reject { |src| linked_out?(src, "guardrails/#{File.basename(src)}") }
58
59
  guardrail_files = install_guardrails(incoming_rules)
59
60
  plugin_file = install_plugin
60
61
  script_files = install_scripts
@@ -81,11 +82,14 @@ module Samagotchi
81
82
  if hook_entries.empty?
82
83
  Dir.glob(File.join(@bundle_dir, "hooks", "*.rb")).each { |p| hook_entries[File.basename(p)] = BundleHook.new }
83
84
  end
84
- hook_entries.each_key do |basename|
85
+ hook_entries.keys.each do |basename|
85
86
  next unless basename.is_a?(String) && !basename.empty?
86
87
 
87
- @bundle_hooks << basename
88
88
  src = File.join(@bundle_dir, "hooks", basename)
89
+ # A linked hook isn't the bundle's: not installed, counted or recorded.
90
+ next hook_entries.delete(basename) if (File.symlink?(src) || File.exist?(src)) && linked_out?(src, "hooks/#{basename}")
91
+
92
+ @bundle_hooks << basename
89
93
  install_hook(basename, src) if File.exist?(src)
90
94
  end
91
95
  end
@@ -120,6 +124,8 @@ module Samagotchi
120
124
  def install_unlisted_hooks
121
125
  Dir.glob(File.join(@bundle_dir, "hooks", "*.rb")).each do |src|
122
126
  basename = File.basename(src)
127
+ next if linked_out?(src, "hooks/#{basename}")
128
+
123
129
  @bundle_hooks << basename
124
130
  if @dry_run
125
131
  @results[basename] = { status: "would_install" }
@@ -207,6 +213,7 @@ module Samagotchi
207
213
 
208
214
  src = File.join(@bundle_dir, plugin[:file])
209
215
  raise Installer::InstallError, "the manifest names plugin #{plugin[:file]}, which the bundle doesn't have" unless File.file?(src)
216
+ raise Installer::InstallError, "plugin #{plugin[:file]} is a symlink (not installed)" if linked?(src)
210
217
 
211
218
  if (failure = Manifest.requires_chi_failure(@manifest.requires_chi, Samagotchi::VERSION))
212
219
  @warnings << "Plugin #{plugin[:file]} won't load: #{failure}"
@@ -243,9 +250,9 @@ module Samagotchi
243
250
  def install_scripts
244
251
  scripts = @manifest&.scripts || {}
245
252
  scripts.each_key do |file|
246
- next if File.file?(File.join(@bundle_dir, "scripts", file))
247
-
248
- raise Installer::InstallError, "the manifest names scripts/#{file}, which the bundle doesn't have"
253
+ src = File.join(@bundle_dir, "scripts", file)
254
+ raise Installer::InstallError, "the manifest names scripts/#{file}, which the bundle doesn't have" unless File.file?(src)
255
+ raise Installer::InstallError, "scripts/#{file} is a symlink (not installed)" if linked?(src)
249
256
  end
250
257
  return {} if @dry_run
251
258
 
@@ -274,6 +281,28 @@ module Samagotchi
274
281
  end
275
282
  end
276
283
 
284
+ # Whether the bundle file +src+ reads through a symlink: it is one, or
285
+ # it (or its dir, hooks/ say) resolves outside the bundle dir. A tar,
286
+ # zip or git repo keeps links; copying through one would install any
287
+ # file the user can read. MemoryFiles skips linked memories the same
288
+ # way.
289
+ def linked?(src)
290
+ return true if File.symlink?(src)
291
+
292
+ root = File.realpath(@bundle_dir)
293
+ !File.realpath(src).start_with?("#{root}/")
294
+ rescue SystemCallError
295
+ true
296
+ end
297
+
298
+ # linked? with a "<label> is a symlink (not installed)" warning.
299
+ def linked_out?(src, label)
300
+ return false unless linked?(src)
301
+
302
+ @warnings << "#{label} is a symlink (not installed)"
303
+ true
304
+ end
305
+
277
306
  # The hook loader skips every hook of a bundle whose requires_chi this
278
307
  # chi doesn't meet (Hooks::BundleLoader); say so at install, as
279
308
  # install_plugin does for the plugin.
@@ -110,6 +110,8 @@ module Samagotchi
110
110
  def select_memories(target_dir, name)
111
111
  candidates = Dir.glob(File.join(target_dir, "*.md")).sort
112
112
  .reject { |p| File.basename(p) == "index.md" || File.basename(p).start_with?(".") }
113
+ symlinks, candidates = candidates.partition { |path| File.symlink?(path) }
114
+ @warnings.concat(symlinks.map { |path| "#{File.basename(path)} is a symlink (not included)" })
113
115
  if candidates.empty?
114
116
  raise BuildError, "No memories to build in #{@scope} scope (#{target_dir}): no .md files found"
115
117
  end
@@ -398,11 +400,10 @@ module Samagotchi
398
400
  raise BuildError, "Output file already exists: #{out_path} — remove it or choose different --out"
399
401
  end
400
402
 
401
- # Use zip CLI like source.rb does with unzip
402
- Dir.chdir(staging) do
403
- result = system("zip", "-r", out_path, ".")
404
- raise BuildError, "zip failed for #{out_path} (is zip installed?)" unless result && File.exist?(out_path)
405
- end
403
+ # Use zip CLI like source.rb does with unzip. It runs in the staging
404
+ # dir (chdir:): Dir.chdir would move the whole process.
405
+ result = system("zip", "-r", out_path, ".", chdir: staging)
406
+ raise BuildError, "zip failed for #{out_path} (is zip installed?)" unless result && File.exist?(out_path)
406
407
  end
407
408
 
408
409
  def tar_staging(staging, out_path, format)
@@ -411,15 +412,13 @@ module Samagotchi
411
412
  raise BuildError, "Output file already exists: #{out_path} — remove it or choose different --out"
412
413
  end
413
414
 
414
- # Use tar CLI
415
- Dir.chdir(staging) do
416
- cmd = case format
417
- when :tar_gz, :tgz then ["tar", "-czf", out_path, "."]
418
- when :tar then ["tar", "-cf", out_path, "."]
419
- end
420
- result = system(*cmd)
421
- raise BuildError, "tar failed for #{out_path}" unless result && File.exist?(out_path)
422
- end
415
+ # Use tar CLI, in the staging dir (chdir:, as zip_staging).
416
+ cmd = case format
417
+ when :tar_gz, :tgz then ["tar", "-czf", out_path, "."]
418
+ when :tar then ["tar", "-cf", out_path, "."]
419
+ end
420
+ result = system(*cmd, chdir: staging)
421
+ raise BuildError, "tar failed for #{out_path}" unless result && File.exist?(out_path)
423
422
  end
424
423
  end
425
424
  end
@@ -34,8 +34,16 @@ module Samagotchi
34
34
 
35
35
  attr_reader :results, :warnings, :placeholder_warnings, :conflicts
36
36
 
37
- def initialize(source:, name:, scope: nil, force: false, strict: true, upgrade: false, dry_run: false)
37
+ # prepared: the source already normalized (a PreparedSource, which the
38
+ # Installer then owns and cleans up as its own); nil normalizes source.
39
+ # shipped_dir: the bundles this chi ships (a bundle of their names
40
+ # installs from a dir in it only, unless force); ShippedUpdate,
41
+ # Profile and SystemBundle pass the one they install from.
42
+ def initialize(source:, name:, scope: nil, force: false, strict: true, upgrade: false, dry_run: false, prepared: nil,
43
+ shipped_dir: SourceNormalizer::SHIPPED_DIR)
38
44
  @source = source
45
+ @shipped_dir = shipped_dir
46
+ @prepared = prepared
39
47
  @name = name
40
48
  @scope = scope&.to_s&.strip&.downcase || ""
41
49
  @force = force
@@ -50,6 +58,10 @@ module Samagotchi
50
58
  @trash = Trash.new(name)
51
59
  end
52
60
 
61
+ # [recorded source, this source] when an upgrade switches the bundle's
62
+ # source (Installer#check_source), else nil.
63
+ attr_reader :source_change
64
+
53
65
  # Where an upgrade moved the files the bundle no longer ships (nil:
54
66
  # none moved).
55
67
  def trash_dir = @trash.dir
@@ -78,6 +90,7 @@ module Samagotchi
78
90
  @file_descriptions = @manifest&.file_descriptions || {}
79
91
  @provenance = Provenance.new(name: @name)
80
92
  @existing = @provenance.record
93
+ check_source
81
94
  # A plain install over an installed bundle: summary points at upgrade.
82
95
  @reinstall = @existing && !@upgrade && !@force && !@dry_run
83
96
  install_memories
@@ -114,6 +127,7 @@ module Samagotchi
114
127
 
115
128
  def summary
116
129
  lines = []
130
+ lines << "Source changes: #{@source_change.join(" → ")}" if @source_change
117
131
  installed = @results.select { |_, r| r[:status].to_s == "installed" }.keys
118
132
  updated = @results.select { |_, r| r[:status].to_s == "updated" }.keys
119
133
  kept = @results.select { |_, r| %w[kept kept_pruned].include?(r[:status].to_s) }.keys
@@ -142,12 +156,90 @@ module Samagotchi
142
156
 
143
157
  private
144
158
 
159
+ # Who may install under this name (a manifest can take any name: a
160
+ # repo calling itself guardrails). Sources compare by SourceIdentity
161
+ # (a shipped dir by name, in any gem or checkout; a git URL without
162
+ # #ref; a path resolved); one it can't place (an older record's
163
+ # relative path) never refuses.
164
+ # - a name chi ships comes from a shipped dir only, unless --force
165
+ # (install and upgrade): the plugin and hooks it replaces are chi's;
166
+ # - a plain install over a bundle from another source is refused,
167
+ # --force too (that overwrites files, it doesn't switch sources):
168
+ # it would delete the plugin and hooks and rewrite the record;
169
+ # - an upgrade may switch sources (a URL and a local copy of it are
170
+ # one bundle, an archive's file name has its version), and
171
+ # #source_change says so (the summary's first line).
172
+ def check_source
173
+ refuse_case_twin
174
+ refuse_shipped_name
175
+ now = incoming_identity
176
+ return if now.nil?
177
+
178
+ was = @existing && SourceIdentity.of(@existing.source)
179
+ return if was.nil? || was == now
180
+ return @source_change = [@existing.source, @source.to_s] if @upgrade
181
+
182
+ raise InstallError, "#{@name} is installed from #{@existing.source}; `chi bundle upgrade #{@source}` " \
183
+ "switches its source, or `chi bundle uninstall #{@name}` first"
184
+ end
185
+
186
+ # A name that differs from an installed bundle's only in case is that
187
+ # bundle's dir on a case-insensitive file system (macOS): BTW would
188
+ # replace btw. Refused, --force too.
189
+ def refuse_case_twin
190
+ twin, = Provenance.each_installed.find { |name, _| name != @name && name.casecmp?(@name) }
191
+ raise InstallError, "#{@name} differs only in case from the installed #{twin}" if twin
192
+ end
193
+
194
+ # A shipped name (in any case) from anything but this chi's own
195
+ # shipped dir. A recorded source may be an older gem's (SourceIdentity
196
+ # matches the path); an incoming one must be a dir in SHIPPED_DIR, so
197
+ # a fork's lib/samagotchi/bundles/btw isn't chi's.
198
+ def refuse_shipped_name
199
+ return if @force || chi_copy?(@source)
200
+
201
+ shipped = shipped_names.find { |name| name.casecmp?(@name) }
202
+ return unless shipped
203
+
204
+ what = shipped == @name ? "#{@name} is a bundle shipped with chi" : "#{@name} is #{shipped}, a bundle shipped with chi"
205
+ raise InstallError, "#{what}; #{@source} isn't chi's copy of it (--force to install it anyway)"
206
+ end
207
+
208
+ # This source's SourceIdentity; one that looks shipped by its path (a
209
+ # fork's lib/samagotchi/bundles/btw) but isn't a dir in the shipped
210
+ # dir is its path, so it isn't taken for chi's btw. A recorded source
211
+ # keeps the path match (older gems).
212
+ def incoming_identity
213
+ identity = SourceIdentity.of(@source)
214
+ identity&.shipped? && !chi_copy?(@source) ? SourceIdentity.path(@source) : identity
215
+ end
216
+
217
+ # Whether +source+ is a dir in the shipped dir.
218
+ def chi_copy?(source)
219
+ path = File.realpath(source.to_s)
220
+ File.directory?(path) && File.dirname(path) == File.realpath(@shipped_dir)
221
+ rescue SystemCallError
222
+ false
223
+ end
224
+
225
+ # The names of the bundles shipped with chi (their manifests'),
226
+ # samagotchi-system and profiles included.
227
+ def shipped_names
228
+ Dir.glob(File.join(@shipped_dir, "*", "manifest.yml")).filter_map do |path|
229
+ Manifest.read(dir: File.dirname(path)).name
230
+ rescue Manifest::ValidationError
231
+ nil
232
+ end
233
+ end
234
+
145
235
  # The source normalized to a dir (@bundle_dir; an owned one, extracted
146
236
  # from git/zip/tar, is cleaned up after #run) and its manifest.yml
147
237
  # (@manifest, nil when it has none and the install isn't strict).
148
238
  def load_source
149
- @bundle_dir, @source_owned, @source_commit = SourceNormalizer.normalize(@source)
150
- @source_dir = @bundle_dir
239
+ prepared = @prepared || PreparedSource.prepare(@source)
240
+ @bundle_dir = @source_dir = prepared.dir
241
+ @source_owned = prepared.owned
242
+ @source_commit = prepared.commit
151
243
  @manifest = Manifest.read(dir: @bundle_dir)
152
244
  rescue SourceNormalizer::UnknownSourceError => e
153
245
  raise InstallError, "Source normalization failed: #{e.message}"
@@ -181,6 +273,7 @@ module Samagotchi
181
273
  @bundle_md = []
182
274
  @owned = []
183
275
  warn_unshipped_files
276
+ warn_symlinked_files
184
277
  memory_files.each do |file_key|
185
278
  file_path = File.join(@bundle_dir, file_key)
186
279
  @bundle_md << file_key
@@ -220,6 +313,11 @@ module Samagotchi
220
313
  @warnings << "files: names #{missing.join(", ")}, which the bundle doesn't ship (not installed)" unless missing.empty?
221
314
  end
222
315
 
316
+ # A memory that is a symlink isn't installed (MemoryFiles.symlinks).
317
+ def warn_symlinked_files
318
+ MemoryFiles.symlinks(@bundle_dir, @manifest).each { |name| @warnings << "#{name} is a symlink (not installed)" }
319
+ end
320
+
223
321
  # An upgrade's 3-way merge of one file (Merger.classify against the
224
322
  # base the last install recorded); a conflict leaves the local file
225
323
  # and is recorded in @conflicts for the agent step.
@@ -356,13 +454,32 @@ module Samagotchi
356
454
  # for a local edit: keep the installed base, so the next upgrade still
357
455
  # sees the edit (not a fast-forward over it).
358
456
  def install_provenance_files
359
- @owned.each_with_object({}) do |file_key, files|
457
+ files = @owned.each_with_object({}) do |file_key, map|
360
458
  target_path = File.join(@target_dir, file_key)
361
459
  next unless File.exist?(target_path)
362
460
 
363
461
  base_path = @provenance.base_path(file_key)
364
462
  kept = @existing && @results.dig(file_key, :reason) == "already exists" && File.exist?(base_path)
365
- files[file_key] = kept ? base_path : target_path
463
+ map[file_key] = kept ? base_path : target_path
464
+ end
465
+ files.merge(dropped_owned_files)
466
+ end
467
+
468
+ # file key => its base, for each file an earlier install owned that
469
+ # this version doesn't ship and that is still on disk. A plain
470
+ # install leaves such a file where it is, and keeps its record, so the
471
+ # upgrade the summary points at still moves it to the trash
472
+ # (prune_dropped_files); dropping the record orphaned it as a memory.
473
+ def dropped_owned_files
474
+ return {} unless @existing
475
+
476
+ @existing.files.each_key.with_object({}) do |key, files|
477
+ file_key = key.to_s
478
+ next if @bundle_md.include?(file_key)
479
+ next unless File.exist?(File.join(@target_dir, file_key))
480
+
481
+ base_path = @provenance.base_path(file_key)
482
+ files[file_key] = base_path if File.exist?(base_path)
366
483
  end
367
484
  end
368
485
 
@@ -64,8 +64,13 @@ module Samagotchi
64
64
 
65
65
  def initialize(path:)
66
66
  @path = path
67
- raw = YAML.load_file(path) || {}
67
+ raw = load_mapping(path)
68
68
  @name = require_str(raw, "name")
69
+ # The name is a directory under .bundles/: never a path.
70
+ if @name.match?(%r{[/\\\0]}) || %w[. ..].include?(@name)
71
+ raise ValidationError, "name: #{@name.inspect} is not a bundle name (no path separators, not . or ..)"
72
+ end
73
+
69
74
  @version = require_str(raw, "version")
70
75
  @scope = normalize_scope(raw["scope"])
71
76
  @description = (raw["description"] || "").to_s
@@ -188,6 +193,20 @@ module Samagotchi
188
193
 
189
194
  private
190
195
 
196
+ # The manifest.yml's top-level mapping ({} for an empty file); YAML that
197
+ # doesn't parse, or isn't a mapping, raises ValidationError, as every
198
+ # other manifest problem does.
199
+ def load_mapping(path)
200
+ raw = YAML.load_file(path) || {}
201
+ raise ValidationError, "manifest.yml is not a YAML mapping in #{File.dirname(path)}" unless raw.is_a?(Hash)
202
+
203
+ raw
204
+ rescue Psych::SyntaxError => e
205
+ raise ValidationError, "manifest.yml is not valid YAML in #{File.dirname(path)}: line #{e.line}, column #{e.column}: #{e.problem}"
206
+ rescue Psych::Exception => e
207
+ raise ValidationError, "manifest.yml is not valid YAML in #{File.dirname(path)}: #{e.message}"
208
+ end
209
+
191
210
  def require_str(hash, key)
192
211
  val = hash[key]
193
212
  return val if val.is_a?(String) && !val.empty?
@@ -13,6 +13,9 @@ module Samagotchi
13
13
  # files: {} means "no list" rather than "no memories": shipped bundles
14
14
  # with nothing to install write it (mcp), and an external bundle that
15
15
  # left the map empty installed its *.md before this rule.
16
+ # A symlink is never a memory (.symlinks names the ones skipped): a
17
+ # tar, zip or git repo keeps links, and copying through one would put
18
+ # any file the user can read (~/.ssh/..., an API key) into a memory.
16
19
  module MemoryFiles
17
20
  # A repo's meta files, by name without extension (case-insensitive).
18
21
  REPO_META = %w[readme changelog license licence contributing code_of_conduct security].freeze
@@ -22,18 +25,33 @@ module Samagotchi
22
25
 
23
26
  # @return [Array<String>] the memories' basenames, sorted
24
27
  def select(bundle_dir, manifest)
25
- present = Dir.glob(File.join(bundle_dir, "*.md")).map { |f| File.basename(f) }.sort
26
- listed = manifest&.files&.keys
27
- return present & listed if listed && !listed.empty?
28
+ candidates(bundle_dir, manifest).reject { |name| File.symlink?(File.join(bundle_dir, name)) }
29
+ end
28
30
 
29
- present.reject { |name| repo_meta?(name) }
31
+ # The memories .select leaves out because they are symlinks, sorted.
32
+ def symlinks(bundle_dir, manifest)
33
+ candidates(bundle_dir, manifest).select { |name| File.symlink?(File.join(bundle_dir, name)) }
30
34
  end
31
35
 
32
- # The files: entries that name no top-level *.md of the bundle.
36
+ # The files: entries that name no top-level *.md of the bundle (a
37
+ # symlink is in .symlinks, not here).
33
38
  def unshipped(bundle_dir, manifest)
34
39
  listed = manifest&.files&.keys || []
35
- listed.reject { |name| name.end_with?(".md") && File.file?(File.join(bundle_dir, name)) && !name.include?("/") }
40
+ listed.reject do |name|
41
+ path = File.join(bundle_dir, name)
42
+ name.end_with?(".md") && !name.include?("/") && (File.symlink?(path) || File.file?(path))
43
+ end
44
+ end
45
+
46
+ # The top-level *.md files the rules above pick, symlinks included.
47
+ def candidates(bundle_dir, manifest)
48
+ present = Dir.glob(File.join(bundle_dir, "*.md")).map { |f| File.basename(f) }.sort
49
+ listed = manifest&.files&.keys
50
+ return present & listed if listed && !listed.empty?
51
+
52
+ present.reject { |name| repo_meta?(name) }
36
53
  end
54
+ private_class_method :candidates
37
55
 
38
56
  # Whether +name+ is a repo's meta file: README, README.md,
39
57
  # readme.markdown, LICENSE.txt, ...
@@ -109,7 +109,7 @@ module Samagotchi
109
109
  return result.skipped[member] = failure if failure
110
110
  return result.installed << member if dry_run
111
111
 
112
- Installer.new(source: member_dir, name: member, scope: "system", force: force, strict: true).run
112
+ Installer.new(source: member_dir, name: member, scope: "system", force: force, strict: true, shipped_dir: shipped_dir).run
113
113
  result.installed << member
114
114
  rescue StandardError => e
115
115
  result.failed[member] = e.message.lines.first.to_s.strip
@@ -65,7 +65,8 @@ module Samagotchi
65
65
  next row unless row.status == :would_update
66
66
  next apply_profile(row) if row.adds
67
67
 
68
- installer = Installer.new(source: row.source_dir, name: row.name, scope: row.scope, strict: true, upgrade: true)
68
+ installer = Installer.new(source: row.source_dir, name: row.name, scope: row.scope, strict: true, upgrade: true,
69
+ shipped_dir: File.dirname(row.source_dir))
69
70
  installer.run
70
71
  row.dup.tap do |r|
71
72
  r.status = :updated
@@ -50,6 +50,10 @@ module Samagotchi
50
50
  return nil unless name.match?(/\A[A-Za-z0-9][A-Za-z0-9_-]*\z/)
51
51
  return nil if File.exist?(File.join(cwd, name))
52
52
 
53
+ # Dir.children, not File.file? alone: the name in its exact case
54
+ # (a case-insensitive file system finds btw's dir for BTW).
55
+ return nil unless File.directory?(SHIPPED_DIR) && Dir.children(SHIPPED_DIR).include?(name)
56
+
53
57
  dir = File.join(SHIPPED_DIR, name)
54
58
  File.file?(File.join(dir, "manifest.yml")) ? dir : nil
55
59
  end
@@ -128,6 +132,9 @@ module Samagotchi
128
132
  # Copies the contents of `src_dir` into a new permanent directory, stripping
129
133
  # one level of nesting if there's a single top-level entry.
130
134
  # The returned path survives after `src_dir` is cleaned up.
135
+ # A symlink stays a symlink, at the top level too (cp_r dereferences
136
+ # its root by default): the Installer then skips it (MemoryFiles,
137
+ # AssetInstaller#linked?) instead of installing what it points at.
131
138
  def self.clean_copy_of(src_dir)
132
139
  contents = Dir.entries(src_dir).reject { |e| e.start_with?(".") }
133
140
  # Zip-slip check: reject entries with ".." path components.
@@ -139,21 +146,77 @@ module Samagotchi
139
146
 
140
147
  if contents.size == 1
141
148
  entry_path = File.join(src_dir, contents.first)
142
- if File.directory?(entry_path) && !Dir.empty?(entry_path)
149
+ if File.directory?(entry_path) && !File.symlink?(entry_path) && !Dir.empty?(entry_path)
143
150
  dst = Dir.mktmpdir("samagotchi-bundle-")
144
- FileUtils.cp_r("#{entry_path}/.", dst)
151
+ FileUtils.cp_r("#{entry_path}/.", dst, dereference_root: false)
145
152
  return [dst, true]
146
153
  end
147
154
  end
148
155
 
149
156
  dst = Dir.mktmpdir("samagotchi-bundle-")
150
157
  contents.each do |entry|
151
- FileUtils.cp_r(File.join(src_dir, entry), dst)
158
+ FileUtils.cp_r(File.join(src_dir, entry), dst, dereference_root: false)
152
159
  end
153
160
  [dst, true]
154
161
  end
155
162
 
156
163
  private_class_method :normalize_zip, :normalize_tar, :clean_copy_of
157
164
  end
165
+
166
+ # What a bundle's source is, for telling whether two installs of one
167
+ # name come from the same place (Installer: a bundle from elsewhere
168
+ # doesn't silently replace an installed one). key is
169
+ # shipped:<dir> a shipped dir (SHIPPED_SOURCE), in any gem or checkout,
170
+ # so `chi update` and a newer chi upgrade it
171
+ # git:<url> a git URL without its #ref, a trailing / or .git
172
+ # path:<path> an absolute path, symlinks resolved when it exists
173
+ # .of is nil for a source it can't place (empty, or a relative path an
174
+ # older record may hold): unknown never refuses.
175
+ SourceIdentity = Data.define(:key) do
176
+ def self.of(source)
177
+ s = source.to_s.strip
178
+ return nil if s.empty?
179
+
180
+ if SourceNormalizer.git_url?(s)
181
+ url = SourceNormalizer.strip_git_ref(s).first.sub(%r{/+\z}, "").delete_suffix(".git")
182
+ new(key: "git:#{url}")
183
+ elsif s.match?(SourceNormalizer::SHIPPED_SOURCE)
184
+ new(key: "shipped:#{File.basename(s.chomp("/"))}")
185
+ elsif s.start_with?("/")
186
+ new(key: "path:#{real(s)}")
187
+ end
188
+ end
189
+
190
+ # +source+ as a path:<realpath>, whatever it looks like.
191
+ def self.path(source) = new(key: "path:#{real(source.to_s)}")
192
+
193
+ def self.real(path)
194
+ File.realpath(path)
195
+ rescue SystemCallError
196
+ File.expand_path(path)
197
+ end
198
+ private_class_method :real
199
+
200
+ def shipped? = key.start_with?("shipped:")
201
+
202
+ def to_s = key
203
+ end
204
+
205
+ # A source normalized once to a local dir (SourceNormalizer.normalize):
206
+ # owned when it is a git clone or a zip/tar extract, which #cleanup
207
+ # removes; commit is a git clone's HEAD (nil otherwise). BundleCommand
208
+ # reads the bundle's name from it and the Installer installs from it, so
209
+ # a source is cloned or extracted once per command.
210
+ PreparedSource = Data.define(:dir, :owned, :commit) do
211
+ # @raise [SourceNormalizer::UnknownSourceError]
212
+ def self.prepare(source)
213
+ dir, owned, commit = SourceNormalizer.normalize(source)
214
+ new(dir: dir, owned: owned, commit: commit)
215
+ end
216
+
217
+ def cleanup
218
+ SourceNormalizer.cleanup(dir) if owned
219
+ end
220
+ end
158
221
  end
159
222
  end