woods 2.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +94 -7
  3. data/CONTRIBUTING.md +134 -19
  4. data/README.md +1 -1
  5. data/docs/AGENT_GUIDE.md +19 -0
  6. data/docs/AGENT_SETUP.md +22 -2
  7. data/docs/BACKEND_MATRIX.md +7 -0
  8. data/docs/CLIENT_HOOKS.md +6 -0
  9. data/docs/CONFIGURATION_REFERENCE.md +133 -25
  10. data/docs/CONSOLE_MCP_SETUP.md +82 -30
  11. data/docs/EMBEDDING_MODELS.md +16 -19
  12. data/docs/EXTRACTOR_REFERENCE.md +219 -21
  13. data/docs/FAQ.md +11 -25
  14. data/docs/GETTING_STARTED.md +7 -1
  15. data/docs/INCREMENTAL_EXTRACTION.md +261 -19
  16. data/docs/INDEX_LAYOUT.md +5 -0
  17. data/docs/INTERNALS.md +9 -0
  18. data/docs/MCP_HTTP_TRANSPORT.md +20 -15
  19. data/docs/MCP_SERVERS.md +87 -8
  20. data/docs/MCP_TOOL_COOKBOOK.md +13 -55
  21. data/docs/NOTION_INTEGRATION.md +7 -1
  22. data/docs/PUBLISHED_INDEX.md +6 -0
  23. data/docs/README.md +6 -1
  24. data/docs/RETRIEVAL_GUIDE.md +17 -0
  25. data/docs/SOURCE_FRESHNESS.md +157 -5
  26. data/docs/TOKEN_BENCHMARK.md +10 -18
  27. data/docs/TROUBLESHOOTING.md +70 -14
  28. data/docs/UNBLOCKED_INTEGRATION.md +60 -8
  29. data/docs/UPGRADING_TO_2.md +153 -38
  30. data/docs/WATCH_DAEMON.md +97 -14
  31. data/exe/woods-console-mcp +2 -2
  32. data/lib/generators/woods/templates/woods.rb.tt +2 -1
  33. data/lib/tasks/woods.rake +23 -7
  34. data/lib/tasks/woods_checks.rake +2 -2
  35. data/lib/woods/agent_configuration/cli.rb +1 -1
  36. data/lib/woods/agent_configuration/layout.rb +16 -2
  37. data/lib/woods/agent_configuration/plan.rb +13 -3
  38. data/lib/woods/agent_configuration/planner_validation.rb +4 -2
  39. data/lib/woods/agent_configuration/preflight.rb +5 -3
  40. data/lib/woods/builder.rb +17 -57
  41. data/lib/woods/cache/cache_middleware.rb +56 -30
  42. data/lib/woods/chunking/contributor_chunks.rb +119 -0
  43. data/lib/woods/chunking/semantic_chunker.rb +44 -21
  44. data/lib/woods/console/connection_manager.rb +56 -3
  45. data/lib/woods/console/embedded_executor.rb +30 -5
  46. data/lib/woods/console/rack_middleware.rb +29 -1
  47. data/lib/woods/dependency_graph.rb +34 -10
  48. data/lib/woods/embedding/fake.rb +12 -0
  49. data/lib/woods/embedding/indexer.rb +195 -98
  50. data/lib/woods/embedding/input_budget.rb +67 -0
  51. data/lib/woods/embedding/openai.rb +70 -20
  52. data/lib/woods/embedding/provider.rb +37 -25
  53. data/lib/woods/embedding/text_preparer.rb +76 -32
  54. data/lib/woods/embedding/token_counter.rb +18 -81
  55. data/lib/woods/embedding/vector_configuration.rb +48 -0
  56. data/lib/woods/extraction_identities.rb +175 -0
  57. data/lib/woods/extractor.rb +304 -107
  58. data/lib/woods/extractors/action_cable_extractor.rb +8 -3
  59. data/lib/woods/extractors/assigned_value_discovery.rb +74 -0
  60. data/lib/woods/extractors/class_declarations.rb +121 -0
  61. data/lib/woods/extractors/configuration_extractor.rb +11 -3
  62. data/lib/woods/extractors/declaration_ancestry.rb +92 -0
  63. data/lib/woods/extractors/event_extractor.rb +8 -0
  64. data/lib/woods/extractors/graphql_extractor.rb +134 -77
  65. data/lib/woods/extractors/job_extractor.rb +5 -1
  66. data/lib/woods/extractors/lib_extractor.rb +132 -15
  67. data/lib/woods/extractors/mailer_extractor.rb +3 -5
  68. data/lib/woods/extractors/manager_extractor.rb +7 -21
  69. data/lib/woods/extractors/migration_declaration.rb +87 -0
  70. data/lib/woods/extractors/migration_extractor.rb +5 -39
  71. data/lib/woods/extractors/phlex_extractor.rb +6 -2
  72. data/lib/woods/extractors/policy_extractor.rb +9 -5
  73. data/lib/woods/extractors/poro_extractor.rb +112 -53
  74. data/lib/woods/extractors/pundit_extractor.rb +11 -6
  75. data/lib/woods/extractors/scheduled_job_extractor.rb +45 -4
  76. data/lib/woods/extractors/serializer_extractor.rb +34 -22
  77. data/lib/woods/extractors/shared_utility_methods.rb +18 -1
  78. data/lib/woods/extractors/source_nesting.rb +142 -106
  79. data/lib/woods/extractors/standalone_module_discovery.rb +123 -0
  80. data/lib/woods/extractors/state_machine_extractor.rb +46 -40
  81. data/lib/woods/extractors/view_component_extractor.rb +9 -7
  82. data/lib/woods/flow_assembler.rb +4 -1
  83. data/lib/woods/generation.rb +25 -0
  84. data/lib/woods/hooks/context_hint.rb +7 -2
  85. data/lib/woods/mcp/bootstrapper.rb +33 -7
  86. data/lib/woods/mcp/config_resolver.rb +26 -7
  87. data/lib/woods/mcp/index_reader.rb +125 -24
  88. data/lib/woods/mcp/index_reader_pinning.rb +16 -0
  89. data/lib/woods/mcp/renderers/markdown_renderer.rb +7 -1
  90. data/lib/woods/mcp/renderers/plain_renderer.rb +3 -1
  91. data/lib/woods/mcp/search_results.rb +7 -1
  92. data/lib/woods/mcp/server.rb +24 -4
  93. data/lib/woods/module_reconciliation.rb +151 -0
  94. data/lib/woods/path_dispatcher.rb +7 -2
  95. data/lib/woods/rake_helpers.rb +43 -11
  96. data/lib/woods/release.rb +1 -1
  97. data/lib/woods/resilience/index_validator.rb +8 -3
  98. data/lib/woods/resilience/retryable_provider.rb +18 -1
  99. data/lib/woods/resolved_config.rb +68 -8
  100. data/lib/woods/retrieval/context_assembler.rb +3 -3
  101. data/lib/woods/retrieval/lexical_assembler.rb +3 -2
  102. data/lib/woods/retrieval/scope.rb +18 -2
  103. data/lib/woods/retrieval/source_evidence.rb +14 -2
  104. data/lib/woods/source_contributor_validation.rb +78 -0
  105. data/lib/woods/source_contributors.rb +116 -0
  106. data/lib/woods/source_inputs/handoff.rb +37 -0
  107. data/lib/woods/source_inputs/launcher.rb +53 -13
  108. data/lib/woods/source_inputs/manifest.rb +84 -3
  109. data/lib/woods/source_inputs/private_key.rb +44 -12
  110. data/lib/woods/source_inputs/scanner.rb +98 -27
  111. data/lib/woods/source_inputs/scopes.rb +1 -1
  112. data/lib/woods/source_inputs/session.rb +147 -15
  113. data/lib/woods/source_inputs/stable_reader.rb +127 -0
  114. data/lib/woods/source_inputs/status.rb +40 -8
  115. data/lib/woods/source_inputs/verifier.rb +28 -5
  116. data/lib/woods/source_path_encoding.rb +33 -0
  117. data/lib/woods/source_references/cache.rb +284 -0
  118. data/lib/woods/source_references/collector.rb +120 -0
  119. data/lib/woods/source_references/extraction.rb +185 -0
  120. data/lib/woods/source_references/inputs.rb +134 -0
  121. data/lib/woods/source_references/parser_adapter.rb +134 -0
  122. data/lib/woods/source_references/pass.rb +152 -0
  123. data/lib/woods/source_references/prism_adapter.rb +116 -0
  124. data/lib/woods/source_references/registry.rb +178 -0
  125. data/lib/woods/source_references/runtime_lookup.rb +127 -0
  126. data/lib/woods/source_references/value_class.rb +82 -0
  127. data/lib/woods/storage/metadata_store.rb +4 -1
  128. data/lib/woods/storage/qdrant.rb +2 -2
  129. data/lib/woods/unblocked/client.rb +12 -7
  130. data/lib/woods/unblocked/document_builder.rb +4 -1
  131. data/lib/woods/unblocked/exporter.rb +127 -37
  132. data/lib/woods/unblocked/sync_manifest.rb +137 -21
  133. data/lib/woods/unblocked/uri_migration.rb +105 -0
  134. data/lib/woods/util/host_guard.rb +3 -2
  135. data/lib/woods/version.rb +1 -1
  136. data/lib/woods/watch/catch_up.rb +138 -0
  137. data/lib/woods/watch/claim_lease.rb +150 -0
  138. data/lib/woods/watch/cli.rb +26 -2
  139. data/lib/woods/watch/daemon.rb +80 -59
  140. data/lib/woods/watch/installation/options.rb +1 -1
  141. data/lib/woods/watch/installation/receipt.rb +6 -1
  142. data/lib/woods/watch/managed_child.rb +1 -1
  143. data/lib/woods/watch/supervisor.rb +1 -1
  144. data/lib/woods/watch/tree_scan.rb +14 -2
  145. data/plugin/.claude-plugin/plugin.json +1 -1
  146. data/plugin/hooks/adapters/normalize.rb +3 -2
  147. data/plugin/hooks/woods-input-rules.sh +4 -0
  148. data/plugin/hooks/woods-refresh.sh +15 -7
  149. data/plugin/hooks/woods-session-start.sh +60 -3
  150. data/plugin/skills/woods-diagnose/SKILL.md +334 -11
  151. data/plugin/skills/woods-investigate/SKILL.md +11 -0
  152. data/plugin/skills/woods-mcp-config/SKILL.md +79 -8
  153. data/plugin/skills/woods-setup/SKILL.md +53 -6
  154. metadata +32 -5
@@ -8,6 +8,7 @@ require_relative 'client'
8
8
  require_relative 'rate_limiter'
9
9
  require_relative 'document_builder'
10
10
  require_relative 'sync_manifest'
11
+ require_relative 'uri_migration'
11
12
  require_relative '../export/typed_reader'
12
13
 
13
14
  module Woods
@@ -20,7 +21,7 @@ module Woods
20
21
  # remote document_id of everything last pushed, so each run only PUTs
21
22
  # new/changed documents, skips unchanged ones, and deletes documents whose
22
23
  # source unit has disappeared. Documents are upserted by URI, so a missing
23
- # manifest (first run / CI cache miss) degrades to a correct full sync.
24
+ # manifest rebuilds current receipts without adopting remote deletion rights.
24
25
  #
25
26
  # @example
26
27
  # exporter = Exporter.new(index_dir: "tmp/woods")
@@ -61,7 +62,8 @@ module Woods
61
62
  # @param output [IO] Progress output stream (default: $stdout)
62
63
  # @raise [ConfigurationError] if required config is missing
63
64
  def initialize(index_dir:, config: Woods.configuration, client: nil, reader: nil,
64
- manifest: nil, force_full: false, force_purge: false, output: $stdout)
65
+ manifest: nil, force_full: false, force_purge: false, output: $stdout,
66
+ migrate_from_ref: nil, dry_run: false)
65
67
  @collection_id = config.unblocked_collection_id
66
68
  raise ConfigurationError, 'unblocked_collection_id is required' unless @collection_id
67
69
 
@@ -80,7 +82,9 @@ module Woods
80
82
  # Cite the ref the index was actually extracted from. `main` was
81
83
  # hardcoded, so citations on a `master`-default repo pointed at a
82
84
  # branch that need not exist.
83
- @builder = DocumentBuilder.new(repo_url: repo_url, ref: extracted_ref)
85
+ @repo_url = repo_url.chomp('/')
86
+ @migrate_from_ref = migrate_from_ref
87
+ @dry_run = dry_run
84
88
  @manifest = manifest || build_manifest(index_dir)
85
89
  @force_full = force_full
86
90
  @force_purge = force_purge
@@ -99,11 +103,13 @@ module Woods
99
103
  # @return [Hash] { synced:, skipped:, deleted:, errors: }
100
104
  def sync_all
101
105
  prepared = false
106
+ @stats = nil
102
107
  with_prepared_index do
103
108
  prepared = true
104
109
  @current_uris = Set.new
110
+ @attempted_uris = Set.new
105
111
  @budget_exhausted = false
106
- reconcile_from_remote if @manifest.empty?
112
+ return preview if @dry_run
107
113
 
108
114
  synced = 0
109
115
  skipped = 0
@@ -128,11 +134,32 @@ module Woods
128
134
  errors.concat(result[:errors])
129
135
  end
130
136
 
131
- deleted = @budget_exhausted || @ambiguous_uris.any? ? 0 : purge_stale(errors)
132
- { synced: synced, skipped: skipped, deleted: deleted, errors: cap_errors(errors) }
137
+ unless @budget_exhausted || errors.any?
138
+ pending = @migration.plan.map { |move| move['new_uri'] }.compact.to_set - @attempted_uris
139
+ replacements = @published_units.select { |unit| unit['file_path'] && pending.include?(effective_uri(unit)) }
140
+ result = sync_unit_data(replacements.map { |unit| [unit, unit] })
141
+ synced += result[:synced]
142
+ skipped += result[:skipped]
143
+ errors.concat(result[:errors])
144
+ end
145
+
146
+ deleted = 0
147
+ unless @budget_exhausted || @ambiguous_uris.any? || errors.any?
148
+ deleted = purge_stale(errors)
149
+ deleted += @migration.cleanup(current_uris: @current_uris, errors: errors,
150
+ replacement_hashes: migration_hashes, remote_documents: @remote_documents,
151
+ collection_id: @collection_id)
152
+ end
153
+ if @manifest.unresolved_legacy_ownership?
154
+ errors << 'cleanup incomplete: legacy owned documents remain; select their source ref with ' \
155
+ 'UNBLOCKED_MIGRATE_FROM_REF, or review obsolete remote documents and the legacy manifest ' \
156
+ 'receipts for manual cleanup'
157
+ end
158
+ @stats = { synced: synced, skipped: skipped, deleted: deleted, errors: cap_errors(errors),
159
+ complete: errors.empty? && !@budget_exhausted && @migration.plan.empty? }
133
160
  end
134
161
  ensure
135
- save_manifest if prepared
162
+ save_manifest if prepared && !@dry_run
136
163
  end
137
164
 
138
165
  # Sync all units of a given type.
@@ -141,6 +168,8 @@ module Woods
141
168
  # @return [Hash] { synced:, skipped:, errors: }
142
169
  def sync_type(type)
143
170
  with_prepared_index do
171
+ return preview if @dry_run
172
+
144
173
  units = units_for(type)
145
174
  log " #{type}: #{units.size} units"
146
175
 
@@ -155,6 +184,8 @@ module Woods
155
184
  # @return [Hash] { synced:, skipped:, errors: }
156
185
  def sync_type_partial(type, max_count)
157
186
  with_prepared_index do
187
+ return preview if @dry_run
188
+
158
189
  units = units_for(type)
159
190
  units.each { |unit| track_uri(unit) }
160
191
  top = units.sort_by { |unit| -(unit['dependents'] || []).size }.first(max_count)
@@ -173,8 +204,22 @@ module Woods
173
204
  return yield if @prepared_index
174
205
 
175
206
  with_pinned_index do
207
+ @builder = DocumentBuilder.new(repo_url: @repo_url, ref: extracted_ref)
176
208
  @published_units = @typed_reader.all
177
209
  build_uri_index
210
+ @manifest.activate_scope(repo_url: @repo_url, ref: @builder.ref,
211
+ current_uris: @published_units.filter_map do |unit|
212
+ effective_uri(unit) if unit['file_path']
213
+ end)
214
+ @migration = UriMigration.new(manifest: @manifest, client: @client, from_ref: @migrate_from_ref,
215
+ replacements: migration_replacements)
216
+ unless @dry_run
217
+ inventory = @client.all_documents(collection_id: nil)
218
+ @remote_documents = inventory.each_with_object({}) do |doc, documents|
219
+ uri = doc['uri']
220
+ documents[uri] = doc if uri.is_a?(String) && !uri.empty?
221
+ end
222
+ end
178
223
  @prepared_index = true
179
224
  begin
180
225
  yield
@@ -217,6 +262,7 @@ module Woods
217
262
 
218
263
  entries_with_data.each do |entry, unit_data|
219
264
  track_uri(unit_data)
265
+ (@attempted_uris ||= Set.new).add(effective_uri(unit_data)) if unit_data['file_path']
220
266
  if push_document(unit_data) == :skipped
221
267
  skipped += 1
222
268
  else
@@ -252,6 +298,10 @@ module Woods
252
298
  # keeps it, the rest get a `?unit=` suffix so each is a distinct remote
253
299
  # document (and a distinct manifest key).
254
300
  uri = effective_uri(unit_data)
301
+ remote = @remote_documents && @remote_documents[uri]
302
+ if remote && remote['collectionId'] != @collection_id
303
+ raise Woods::ExtractionError, 'export URI belongs to a different remote collection — push refused'
304
+ end
255
305
 
256
306
  doc = @builder.build(unit_data)
257
307
  # An empty body means the credential scrub failed closed (the builders
@@ -262,7 +312,8 @@ module Woods
262
312
  end
263
313
 
264
314
  hash = fingerprint(doc)
265
- return :skipped if !@force_full && @manifest.unchanged?(uri, hash)
315
+ remote_matches = remote && remote['id'] && remote['id'] == @manifest.document_id_for(uri)
316
+ return :skipped if !@force_full && remote_matches && @manifest.unchanged?(uri, hash)
266
317
 
267
318
  response = @client.put_document(
268
319
  collection_id: @collection_id,
@@ -270,7 +321,8 @@ module Woods
270
321
  body: doc[:body],
271
322
  uri: uri
272
323
  )
273
- document_id = (response['id'] if response.is_a?(Hash)) || @manifest.document_id_for(uri)
324
+ document_id = (response['id'] if response.is_a?(Hash)) || remote&.fetch('id', nil)
325
+ @remote_documents[uri] = { 'uri' => uri, 'id' => document_id, 'collectionId' => @collection_id }
274
326
  @manifest.record(uri: uri, hash: hash, document_id: document_id)
275
327
  :synced
276
328
  end
@@ -284,14 +336,31 @@ module Woods
284
336
  def purge_stale(errors)
285
337
  stale = @manifest.stale_uris(@current_uris)
286
338
  return 0 if stale.empty?
287
- return 0 if guard_blocks_purge?(stale)
339
+
340
+ if guard_blocks_purge?(stale)
341
+ errors << 'cleanup incomplete: mass-deletion guard refused stale documents'
342
+ return 0
343
+ end
288
344
 
289
345
  resolve_missing_document_ids(stale)
290
346
 
291
347
  deleted = 0
292
348
  stale.each do |uri|
349
+ remote = @remote_documents[uri]
350
+ unless remote
351
+ @manifest.forget(uri)
352
+ next
353
+ end
293
354
  document_id = @manifest.document_id_for(uri)
294
- next unless document_id
355
+ unless document_id
356
+ errors << "cleanup incomplete: remote document ID unresolved for #{uri}"
357
+ next
358
+ end
359
+
360
+ unless remote['id'] == document_id && remote['collectionId'] == @collection_id
361
+ errors << "cleanup incomplete: remote identity/collection changed for #{uri}"
362
+ next
363
+ end
295
364
 
296
365
  @client.delete_document(document_id: document_id)
297
366
  @manifest.forget(uri)
@@ -324,8 +393,8 @@ module Woods
324
393
  missing = stale.select { |uri| @manifest.document_id_for(uri).nil? }
325
394
  return if missing.empty?
326
395
 
327
- ids_by_uri = @client.all_documents(collection_id: @collection_id)
328
- .to_h { |doc| [doc['uri'], doc['id']] }
396
+ ids_by_uri = @remote_documents.values.select { |doc| doc['collectionId'] == @collection_id }
397
+ .to_h { |doc| [doc['uri'], doc['id']] }
329
398
  missing.each do |uri|
330
399
  id = ids_by_uri[uri]
331
400
  @manifest.record(uri: uri, hash: nil, document_id: id) if id
@@ -352,27 +421,47 @@ module Woods
352
421
  true
353
422
  end
354
423
 
355
- # Seed the manifest from the remote collection when we have no local
356
- # state (first run / CI cache miss). The list endpoint returns no body,
357
- # so hashes are nil (everything re-pushes), but recovering document_ids
358
- # lets this run still purge orphaned documents.
359
- #
360
- # Auth failures re-raise: a 401/403 here dooms every subsequent call,
361
- # and "proceeding with full sync" would burn the whole daily budget on
362
- # guaranteed failures.
363
- def reconcile_from_remote
364
- @client.all_documents(collection_id: @collection_id).each do |doc|
365
- uri = doc['uri']
366
- next unless uri
367
-
368
- @manifest.record(uri: uri, hash: nil, document_id: doc['id'])
424
+ # A preview reads published data and local ownership only. It makes no
425
+ # remote requests and never persists the in-memory migration plan.
426
+ def preview
427
+ { synced: 0, skipped: 0, deleted: 0, errors: [], complete: false, dry_run: true,
428
+ migration: @migration.plan, scope: { repo_url: @repo_url, ref: @builder.ref } }
429
+ end
430
+
431
+ def migration_replacements
432
+ return {} unless @migrate_from_ref
433
+
434
+ legacy_builder = DocumentBuilder.new(repo_url: @repo_url, ref: @migrate_from_ref)
435
+ @published_units.each_with_object({}) do |unit, mapping|
436
+ next unless unit['file_path']
437
+ next if @ambiguous_uris.include?(@builder.uri_for(unit))
438
+
439
+ old_base = legacy_builder.uri_for(unit)
440
+ old_raw = old_base.sub("/blob/#{encode_ref(@migrate_from_ref)}/", "/blob/#{@migrate_from_ref}/")
441
+ suffix = effective_uri(unit).delete_prefix(@builder.uri_for(unit))
442
+ # A v1 bare URI has no typed owner receipt. A new sibling may now
443
+ # own that bare URI; do not guess which historical unit it replaced.
444
+ next if suffix.empty? && @uri_primary.key?(@builder.uri_for(unit))
445
+
446
+ [old_base, old_raw].uniq.each { |base| mapping[base + suffix] = effective_uri(unit) }
369
447
  end
370
- rescue ApiError => e
371
- raise if [401, 403].include?(e.status)
448
+ end
372
449
 
373
- log " reconcile skipped (#{e.message}) — proceeding with full sync"
374
- rescue StandardError => e
375
- log " reconcile skipped (#{e.message}) — proceeding with full sync"
450
+ def migration_hashes
451
+ pending = @migration.plan.map { |move| move['new_uri'] }.compact.to_set
452
+ @published_units.each_with_object({}) do |unit, hashes|
453
+ next unless unit['file_path']
454
+
455
+ uri = effective_uri(unit)
456
+ next unless pending.include?(uri)
457
+
458
+ document = @builder.build(unit)
459
+ hashes[uri] = fingerprint(document) unless document[:body].to_s.empty?
460
+ end
461
+ end
462
+
463
+ def encode_ref(ref)
464
+ ref.split('/').map { |segment| ERB::Util.url_encode(segment) }.join('/')
376
465
  end
377
466
 
378
467
  def track_uri(unit_data)
@@ -457,14 +546,15 @@ module Woods
457
546
  nil
458
547
  end
459
548
 
460
- # Persist the manifest, downgrading failures to a warning: losing the
461
- # manifest only costs a full re-check next run, which must not turn an
462
- # otherwise-successful sync into a crash (this runs from an ensure, where
463
- # a raise would also mask any in-flight exception).
549
+ # Persist receipts without masking an in-flight exception. A failed save
550
+ # leaves completion false: cleanup requires durable ownership evidence.
464
551
  def save_manifest
465
552
  @manifest.save
466
553
  rescue StandardError => e
467
- log " WARNING: sync manifest not persisted (#{e.message}) — next run will re-push all documents"
554
+ message = "sync manifest not persisted: #{e.class}: #{e.message}"
555
+ @stats[:errors] << message if @stats
556
+ @stats[:complete] = false if @stats
557
+ log " WARNING: #{message} — recover local ownership state before cleanup"
468
558
  end
469
559
 
470
560
  def build_manifest(index_dir)
@@ -2,6 +2,8 @@
2
2
 
3
3
  require 'json'
4
4
  require 'fileutils'
5
+ require 'digest'
6
+ require 'erb'
5
7
 
6
8
  require_relative '../atomic_file'
7
9
 
@@ -14,8 +16,8 @@ module Woods
14
16
  # entry records the content hash of the document we last pushed for a URI
15
17
  # plus the remote +document_id+ (needed for deletes). Persisted as JSON
16
18
  # alongside the extraction output and restored across CI runs via the CI
17
- # provider's cache. A missing or corrupt file degrades to "everything is
18
- # new" — a correct (if expensive) full sync that rebuilds the manifest.
19
+ # provider's cache. Missing receipts never establish remote deletion rights.
20
+ # Corrupt or mismatched manifests require recovery before another sync.
19
21
  #
20
22
  # Modeled on the embedding indexer's checkpoint (load JSON → compare
21
23
  # per-key hash → save JSON).
@@ -28,7 +30,7 @@ module Woods
28
30
  # manifest.save
29
31
  #
30
32
  class SyncManifest
31
- VERSION = 1
33
+ VERSION = 2
32
34
 
33
35
  # @param path [String] JSON file path for the manifest
34
36
  # @param collection_id [String] Target collection UUID — a stored manifest
@@ -36,7 +38,10 @@ module Woods
36
38
  def initialize(path:, collection_id:)
37
39
  @path = path
38
40
  @collection_id = collection_id
41
+ @scopes = {}
39
42
  @documents = load
43
+ @legacy_documents = @documents
44
+ @documents = @scopes.dig(@active_scope, 'documents') || @legacy_documents
40
45
  end
41
46
 
42
47
  # @return [Boolean] true when no documents are recorded
@@ -58,7 +63,8 @@ module Woods
58
63
  # @param hash [String, nil] Content hash pushed (nil forces a future re-push)
59
64
  # @param document_id [String, nil] Remote document UUID (for later deletes)
60
65
  def record(uri:, hash:, document_id:)
61
- @documents[uri] = { 'hash' => hash, 'document_id' => document_id }
66
+ owned = !hash.nil? || @documents.dig(uri, 'owned') == true
67
+ @documents[uri] = { 'hash' => hash, 'document_id' => document_id, 'owned' => owned }
62
68
  end
63
69
 
64
70
  # @param uri [String] Document URI
@@ -94,38 +100,145 @@ module Woods
94
100
  payload = JSON.generate(
95
101
  'version' => VERSION,
96
102
  'collection_id' => @collection_id,
97
- 'documents' => @documents
103
+ 'documents' => @legacy_documents,
104
+ 'scopes' => @scopes,
105
+ 'active_scope' => @active_scope
98
106
  )
99
107
  AtomicFile.write(@path, payload)
100
108
  end
101
109
 
110
+ # Select one repository/ref without retiring any other branch's documents.
111
+ # Existing v1 entries are adopted only when they prove a successful write
112
+ # and correspond to a currently published URI in this exact scope. A URI
113
+ # prefix cannot distinguish refs containing slashes from source paths.
114
+ def activate_scope(repo_url:, ref:, current_uris:)
115
+ raise Woods::ConfigurationError, "Unblocked manifest needs recovery: #{@load_error}" if @load_error
116
+
117
+ @repo_url = repo_url.chomp('/')
118
+ @ref = ref
119
+ @active_scope = scope_key(@repo_url, ref)
120
+ @scopes[@active_scope] ||= { 'repo_url' => @repo_url, 'ref' => ref, 'documents' => {} }
121
+ @documents = @scopes.fetch(@active_scope).fetch('documents')
122
+ current_uris.each do |uri|
123
+ entry = @legacy_documents[uri]
124
+ next unless entry && owned?(entry)
125
+
126
+ @documents[uri] ||= entry
127
+ @legacy_documents.delete(uri)
128
+ end
129
+ end
130
+
131
+ # A remaining v1 receipt has no selected repository/ref scope. Report
132
+ # the cleanup obligation rather than claiming that synchronization is done.
133
+ def unresolved_legacy_ownership?
134
+ @legacy_documents.any? { |uri, entry| uri.start_with?("#{@repo_url}/blob/") && owned?(entry) }
135
+ end
136
+
137
+ # Local ownership evidence for an explicitly selected migration source.
138
+ # A remote listing never establishes permission to delete a document.
139
+ def migration_sources(from_ref:)
140
+ key = scope_key(@repo_url, from_ref)
141
+ scoped = @scopes.dig(key, 'documents') || {}
142
+ prefixes = [from_ref, encode_ref(from_ref)].uniq.map { |ref| "#{@repo_url}/blob/#{ref}/" }
143
+ legacy = @legacy_documents.select { |uri, _| prefixes.any? { |prefix| uri.start_with?(prefix) } }
144
+ [[:legacy, legacy], [key, scoped]].flat_map do |location, entries|
145
+ entries.map do |uri, entry|
146
+ entry.merge('old_uri' => uri, 'source_scope' => location.to_s, 'owned' => owned?(entry))
147
+ end
148
+ end
149
+ end
150
+
151
+ # Persist the approved source scope and bounded set of replacement pairs.
152
+ def start_migration(from_ref:, moves:)
153
+ finish_migration
154
+ current = migration
155
+ if current && current['from_ref'] != from_ref
156
+ raise Woods::ConfigurationError, 'Finish the pending Unblocked migration before selecting another source ref'
157
+ end
158
+
159
+ return if current.nil? && moves.empty?
160
+
161
+ @scopes.fetch(@active_scope)['migration'] ||= { 'from_ref' => from_ref, 'moves' => moves }
162
+ end
163
+
164
+ def migration
165
+ @scopes.dig(@active_scope, 'migration')
166
+ end
167
+
168
+ def complete_move(move)
169
+ source = if move.fetch('source_scope') == 'legacy'
170
+ @legacy_documents
171
+ else
172
+ @scopes.dig(move.fetch('source_scope'), 'documents')
173
+ end
174
+ source&.delete(move.fetch('old_uri'))
175
+ migration.fetch('moves').delete(move)
176
+ end
177
+
178
+ # Finish only after every selected replacement and old-copy cleanup.
179
+ def finish_migration
180
+ @scopes.fetch(@active_scope).delete('migration') if migration && migration.fetch('moves').empty?
181
+ end
182
+
102
183
  private
103
184
 
104
- # Load the persisted documents, discarding data from a different
105
- # collection, a different schema version, or an unparseable/unreadable
106
- # file. Every discard warns to stderr — the consequence (a full re-push)
107
- # is expensive enough that operators need to know why it happened.
108
- #
109
- # AtomicFile.read, not File.read: a bare read tags the bytes with the
110
- # process's default external encoding (US-ASCII under LANG=C), so a
111
- # non-ASCII document URI raised EncodingError out of JSON.parse instead
112
- # of degrading. Read failures (e.g. Errno::EACCES) take the same discard
113
- # path — the manifest is a cache, and "everything is new" is always a
114
- # correct answer.
115
- #
185
+ def scope_key(repo, ref)
186
+ Digest::SHA256.hexdigest(JSON.generate([repo, ref]))
187
+ end
188
+
189
+ def encode_ref(ref)
190
+ ref.split('/').map { |segment| ERB::Util.url_encode(segment) }.join('/')
191
+ end
192
+
193
+ def owned?(entry)
194
+ entry['owned'] == true || (entry['hash'].is_a?(String) && !entry['hash'].empty?)
195
+ end
196
+
197
+ def valid_documents?(documents)
198
+ documents.is_a?(Hash) && documents.all? { |uri, entry| uri.is_a?(String) && entry.is_a?(Hash) }
199
+ end
200
+
201
+ def valid_scopes?(scopes)
202
+ scopes.is_a?(Hash) && scopes.all? do |key, scope|
203
+ key.is_a?(String) && scope.is_a?(Hash) && scope['repo_url'].is_a?(String) &&
204
+ scope['ref'].is_a?(String) && key == scope_key(scope['repo_url'], scope['ref']) &&
205
+ valid_documents?(scope['documents']) && valid_migration?(scope['migration'])
206
+ end
207
+ end
208
+
209
+ def valid_migration?(migration)
210
+ return true if migration.nil?
211
+ unless migration.is_a?(Hash) && migration['from_ref'].is_a?(String) && migration['moves'].is_a?(Array)
212
+ return false
213
+ end
214
+
215
+ migration['moves'].all? do |move|
216
+ move.is_a?(Hash) && move['old_uri'].is_a?(String) && move['source_scope'].is_a?(String) &&
217
+ (move['new_uri'].nil? || move['new_uri'].is_a?(String)) && [true, false].include?(move['owned'])
218
+ end
219
+ end
220
+
221
+ # Retain legacy records, but refuse malformed ownership state at sync
222
+ # preflight. AtomicFile supplies UTF-8 independently of the host locale.
116
223
  # @return [Hash{String=>Hash}] uri => { 'hash' =>, 'document_id' => }
117
224
  def load
118
225
  return {} unless File.exist?(@path)
119
226
 
120
227
  parsed = JSON.parse(AtomicFile.read(@path))
121
228
  return discard('not a JSON object') unless parsed.is_a?(Hash)
122
- return discard("schema version #{parsed['version'].inspect}, expected #{VERSION}") unless
123
- parsed['version'] == VERSION
229
+ return discard("schema version #{parsed['version'].inspect}, expected 1 or #{VERSION}") unless
230
+ [1, VERSION].include?(parsed['version'])
124
231
  return discard("written for collection #{parsed['collection_id'].inspect}, expected #{@collection_id}") unless
125
232
  parsed['collection_id'] == @collection_id
126
233
 
234
+ @scopes = parsed.fetch('scopes', {})
235
+ return discard('invalid scoped ownership records') unless valid_scopes?(@scopes)
236
+
237
+ @active_scope = parsed['active_scope']
127
238
  documents = parsed['documents']
128
- documents.is_a?(Hash) ? documents : {}
239
+ return discard('invalid document ownership records') unless valid_documents?(documents)
240
+
241
+ documents
129
242
  rescue JSON::ParserError
130
243
  discard('unparseable JSON')
131
244
  rescue EncodingError, SystemCallError => e
@@ -135,7 +248,10 @@ module Woods
135
248
  # @param reason [String] Why the persisted manifest is unusable
136
249
  # @return [Hash] empty documents hash (degrades to a full re-push)
137
250
  def discard(reason)
138
- warn "WARNING: discarding sync manifest at #{@path} (#{reason}) — next sync re-pushes all documents"
251
+ @load_error = reason
252
+ @scopes = {}
253
+ warn "WARNING: discarding sync manifest at #{@path} (#{reason}) — " \
254
+ 'sync requires manifest recovery before remote changes'
139
255
  {}
140
256
  end
141
257
  end
@@ -0,0 +1,105 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Woods
4
+ module Unblocked
5
+ # Resumable, explicitly selected URI replacements. Ordinary branch changes
6
+ # never enter this path and never retire another branch's ownership scope.
7
+ class UriMigration
8
+ # @param manifest [SyncManifest] currently selected target scope
9
+ # @param client [Client] remote document operations
10
+ # @param from_ref [String, nil] explicit source branch, or resume pending work
11
+ # @param replacements [Hash{String => String}] old URI to published target URI
12
+ def initialize(manifest:, client:, from_ref:, replacements:)
13
+ @manifest = manifest
14
+ @client = client
15
+ return unless from_ref
16
+
17
+ moves = manifest.migration_sources(from_ref: from_ref).filter_map do |entry|
18
+ target = replacements[entry.fetch('old_uri')]
19
+ next if target == entry.fetch('old_uri')
20
+
21
+ entry.merge('new_uri' => target)
22
+ end
23
+ manifest.start_migration(from_ref: from_ref, moves: moves)
24
+ end
25
+
26
+ # @return [Array<Hash>] exact pending replacements, suitable for preview
27
+ def plan
28
+ @manifest.migration&.fetch('moves') || []
29
+ end
30
+
31
+ # Save successful replacement receipts before deleting any old copies.
32
+ # This explicit migration set has its own ownership checks; the ordinary
33
+ # mass-delete ratio does not mistake a one-for-one migration for loss.
34
+ # @param current_uris [Set<String>] verified current publication membership
35
+ # @param errors [Array<String>] actionable refusal/failure messages
36
+ # @return [Integer] old copies removed
37
+ def cleanup(current_uris:, errors:, replacement_hashes:, remote_documents:, collection_id:)
38
+ return 0 if plan.empty?
39
+
40
+ @manifest.save
41
+ deleted = 0
42
+ plan.dup.each do |move|
43
+ remote = remote_documents[move.fetch('old_uri')]
44
+ if remote.nil?
45
+ finish_move(move)
46
+ next
47
+ end
48
+ reason = refusal(move, current_uris, replacement_hashes)
49
+ target = remote_documents[move['new_uri']]
50
+ unless target && target['id'] == @manifest.document_id_for(move['new_uri']) &&
51
+ target['collectionId'] == collection_id
52
+ reason ||= 'replacement remote identity is missing or changed'
53
+ end
54
+ reason ||= 'source remote identity/collection changed; review before cleanup' unless
55
+ remote['id'] == move['document_id'] && remote['collectionId'] == collection_id
56
+ if reason
57
+ errors << "URI migration incomplete: #{reason} (#{move.fetch('old_uri')})"
58
+ next
59
+ end
60
+ @client.delete_document(document_id: move.fetch('document_id'))
61
+ deleted += 1
62
+ finish_move(move)
63
+ rescue ApiError => e
64
+ if e.status == 404
65
+ finish_move(move)
66
+ else
67
+ errors << "URI migration cleanup failed: #{e.message}"
68
+ end
69
+ rescue StandardError => e
70
+ errors << "URI migration cleanup incomplete: #{e.class}: #{e.message}"
71
+ break
72
+ end
73
+ @manifest.finish_migration
74
+ deleted
75
+ end
76
+
77
+ private
78
+
79
+ def refusal(move, current_uris, replacement_hashes)
80
+ unless move['owned'] == true
81
+ return 'legacy ownership is unverified; review the remote record before manual cleanup'
82
+ end
83
+ unless move['document_id'].is_a?(String) && !move['document_id'].empty?
84
+ return 'source document ID is unresolved'
85
+ end
86
+ return 'no unambiguous current replacement' unless current_uris.include?(move['new_uri'])
87
+
88
+ expected = replacement_hashes[move['new_uri']]
89
+ return 'replacement content has not been successfully synchronized' unless
90
+ expected && @manifest.unchanged?(move['new_uri'], expected)
91
+
92
+ target_id = @manifest.document_id_for(move['new_uri'])
93
+ return 'replacement has no persisted successful document receipt' unless target_id
94
+ return 'source and replacement document IDs are identical' if target_id == move['document_id']
95
+
96
+ nil
97
+ end
98
+
99
+ def finish_move(move)
100
+ @manifest.complete_move(move)
101
+ @manifest.save
102
+ end
103
+ end
104
+ end
105
+ end
@@ -2,8 +2,9 @@
2
2
 
3
3
  module Woods
4
4
  module Util
5
- # Shared host-header / URL-host canonicalization used by {MCP::OriginGuard}
6
- # and the {Storage::VectorStore::Qdrant} URL validator.
5
+ # Numeric-host checks shared by {MCP::OriginPolicy} and the
6
+ # {Storage::VectorStore::Qdrant} URL validator. Qdrant also uses the URL
7
+ # canonicalization helper; HTTP authorities retain SDK matching semantics.
7
8
  #
8
9
  # Both components need to reject numeric IPv4 notations that `URI` and
9
10
  # `getaddrinfo` accept but `IPAddr` does not — hex (`0x7f000001`),
data/lib/woods/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Woods
4
- VERSION = '2.0.1'
4
+ VERSION = '2.1.0'
5
5
  end