woods 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +94 -7
- data/CONTRIBUTING.md +134 -19
- data/README.md +1 -1
- data/docs/AGENT_GUIDE.md +19 -0
- data/docs/AGENT_SETUP.md +22 -2
- data/docs/BACKEND_MATRIX.md +7 -0
- data/docs/CLIENT_HOOKS.md +6 -0
- data/docs/CONFIGURATION_REFERENCE.md +133 -25
- data/docs/CONSOLE_MCP_SETUP.md +82 -30
- data/docs/EMBEDDING_MODELS.md +16 -19
- data/docs/EXTRACTOR_REFERENCE.md +219 -21
- data/docs/FAQ.md +11 -25
- data/docs/GETTING_STARTED.md +7 -1
- data/docs/INCREMENTAL_EXTRACTION.md +261 -19
- data/docs/INDEX_LAYOUT.md +5 -0
- data/docs/INTERNALS.md +9 -0
- data/docs/MCP_HTTP_TRANSPORT.md +20 -15
- data/docs/MCP_SERVERS.md +87 -8
- data/docs/MCP_TOOL_COOKBOOK.md +13 -55
- data/docs/NOTION_INTEGRATION.md +7 -1
- data/docs/PUBLISHED_INDEX.md +6 -0
- data/docs/README.md +6 -1
- data/docs/RETRIEVAL_GUIDE.md +17 -0
- data/docs/SOURCE_FRESHNESS.md +157 -5
- data/docs/TOKEN_BENCHMARK.md +10 -18
- data/docs/TROUBLESHOOTING.md +70 -14
- data/docs/UNBLOCKED_INTEGRATION.md +60 -8
- data/docs/UPGRADING_TO_2.md +153 -38
- data/docs/WATCH_DAEMON.md +97 -14
- data/exe/woods-console-mcp +2 -2
- data/lib/generators/woods/templates/woods.rb.tt +2 -1
- data/lib/tasks/woods.rake +23 -7
- data/lib/tasks/woods_checks.rake +2 -2
- data/lib/woods/agent_configuration/cli.rb +1 -1
- data/lib/woods/agent_configuration/layout.rb +16 -2
- data/lib/woods/agent_configuration/plan.rb +13 -3
- data/lib/woods/agent_configuration/planner_validation.rb +4 -2
- data/lib/woods/agent_configuration/preflight.rb +5 -3
- data/lib/woods/builder.rb +17 -57
- data/lib/woods/cache/cache_middleware.rb +56 -30
- data/lib/woods/chunking/contributor_chunks.rb +119 -0
- data/lib/woods/chunking/semantic_chunker.rb +44 -21
- data/lib/woods/console/connection_manager.rb +56 -3
- data/lib/woods/console/embedded_executor.rb +30 -5
- data/lib/woods/console/rack_middleware.rb +29 -1
- data/lib/woods/dependency_graph.rb +34 -10
- data/lib/woods/embedding/fake.rb +12 -0
- data/lib/woods/embedding/indexer.rb +195 -98
- data/lib/woods/embedding/input_budget.rb +67 -0
- data/lib/woods/embedding/openai.rb +70 -20
- data/lib/woods/embedding/provider.rb +37 -25
- data/lib/woods/embedding/text_preparer.rb +76 -32
- data/lib/woods/embedding/token_counter.rb +18 -81
- data/lib/woods/embedding/vector_configuration.rb +48 -0
- data/lib/woods/extraction_identities.rb +175 -0
- data/lib/woods/extractor.rb +304 -107
- data/lib/woods/extractors/action_cable_extractor.rb +8 -3
- data/lib/woods/extractors/assigned_value_discovery.rb +74 -0
- data/lib/woods/extractors/class_declarations.rb +121 -0
- data/lib/woods/extractors/configuration_extractor.rb +11 -3
- data/lib/woods/extractors/declaration_ancestry.rb +92 -0
- data/lib/woods/extractors/event_extractor.rb +8 -0
- data/lib/woods/extractors/graphql_extractor.rb +134 -77
- data/lib/woods/extractors/job_extractor.rb +5 -1
- data/lib/woods/extractors/lib_extractor.rb +132 -15
- data/lib/woods/extractors/mailer_extractor.rb +3 -5
- data/lib/woods/extractors/manager_extractor.rb +7 -21
- data/lib/woods/extractors/migration_declaration.rb +87 -0
- data/lib/woods/extractors/migration_extractor.rb +5 -39
- data/lib/woods/extractors/phlex_extractor.rb +6 -2
- data/lib/woods/extractors/policy_extractor.rb +9 -5
- data/lib/woods/extractors/poro_extractor.rb +112 -53
- data/lib/woods/extractors/pundit_extractor.rb +11 -6
- data/lib/woods/extractors/scheduled_job_extractor.rb +45 -4
- data/lib/woods/extractors/serializer_extractor.rb +34 -22
- data/lib/woods/extractors/shared_utility_methods.rb +18 -1
- data/lib/woods/extractors/source_nesting.rb +142 -106
- data/lib/woods/extractors/standalone_module_discovery.rb +123 -0
- data/lib/woods/extractors/state_machine_extractor.rb +46 -40
- data/lib/woods/extractors/view_component_extractor.rb +9 -7
- data/lib/woods/flow_assembler.rb +4 -1
- data/lib/woods/generation.rb +25 -0
- data/lib/woods/hooks/context_hint.rb +7 -2
- data/lib/woods/mcp/bootstrapper.rb +33 -7
- data/lib/woods/mcp/config_resolver.rb +26 -7
- data/lib/woods/mcp/index_reader.rb +125 -24
- data/lib/woods/mcp/index_reader_pinning.rb +16 -0
- data/lib/woods/mcp/renderers/markdown_renderer.rb +7 -1
- data/lib/woods/mcp/renderers/plain_renderer.rb +3 -1
- data/lib/woods/mcp/search_results.rb +7 -1
- data/lib/woods/mcp/server.rb +24 -4
- data/lib/woods/module_reconciliation.rb +151 -0
- data/lib/woods/path_dispatcher.rb +7 -2
- data/lib/woods/rake_helpers.rb +43 -11
- data/lib/woods/release.rb +1 -1
- data/lib/woods/resilience/index_validator.rb +8 -3
- data/lib/woods/resilience/retryable_provider.rb +18 -1
- data/lib/woods/resolved_config.rb +68 -8
- data/lib/woods/retrieval/context_assembler.rb +3 -3
- data/lib/woods/retrieval/lexical_assembler.rb +3 -2
- data/lib/woods/retrieval/scope.rb +18 -2
- data/lib/woods/retrieval/source_evidence.rb +14 -2
- data/lib/woods/source_contributor_validation.rb +78 -0
- data/lib/woods/source_contributors.rb +116 -0
- data/lib/woods/source_inputs/handoff.rb +37 -0
- data/lib/woods/source_inputs/launcher.rb +53 -13
- data/lib/woods/source_inputs/manifest.rb +84 -3
- data/lib/woods/source_inputs/private_key.rb +44 -12
- data/lib/woods/source_inputs/scanner.rb +98 -27
- data/lib/woods/source_inputs/scopes.rb +1 -1
- data/lib/woods/source_inputs/session.rb +147 -15
- data/lib/woods/source_inputs/stable_reader.rb +127 -0
- data/lib/woods/source_inputs/status.rb +40 -8
- data/lib/woods/source_inputs/verifier.rb +28 -5
- data/lib/woods/source_path_encoding.rb +33 -0
- data/lib/woods/source_references/cache.rb +284 -0
- data/lib/woods/source_references/collector.rb +120 -0
- data/lib/woods/source_references/extraction.rb +185 -0
- data/lib/woods/source_references/inputs.rb +134 -0
- data/lib/woods/source_references/parser_adapter.rb +134 -0
- data/lib/woods/source_references/pass.rb +152 -0
- data/lib/woods/source_references/prism_adapter.rb +116 -0
- data/lib/woods/source_references/registry.rb +178 -0
- data/lib/woods/source_references/runtime_lookup.rb +127 -0
- data/lib/woods/source_references/value_class.rb +82 -0
- data/lib/woods/storage/metadata_store.rb +4 -1
- data/lib/woods/storage/qdrant.rb +2 -2
- data/lib/woods/unblocked/client.rb +12 -7
- data/lib/woods/unblocked/document_builder.rb +4 -1
- data/lib/woods/unblocked/exporter.rb +127 -37
- data/lib/woods/unblocked/sync_manifest.rb +137 -21
- data/lib/woods/unblocked/uri_migration.rb +105 -0
- data/lib/woods/util/host_guard.rb +3 -2
- data/lib/woods/version.rb +1 -1
- data/lib/woods/watch/catch_up.rb +138 -0
- data/lib/woods/watch/claim_lease.rb +150 -0
- data/lib/woods/watch/cli.rb +26 -2
- data/lib/woods/watch/daemon.rb +80 -59
- data/lib/woods/watch/installation/options.rb +1 -1
- data/lib/woods/watch/installation/receipt.rb +6 -1
- data/lib/woods/watch/managed_child.rb +1 -1
- data/lib/woods/watch/supervisor.rb +1 -1
- data/lib/woods/watch/tree_scan.rb +14 -2
- data/plugin/.claude-plugin/plugin.json +1 -1
- data/plugin/hooks/adapters/normalize.rb +3 -2
- data/plugin/hooks/woods-input-rules.sh +4 -0
- data/plugin/hooks/woods-refresh.sh +15 -7
- data/plugin/hooks/woods-session-start.sh +60 -3
- data/plugin/skills/woods-diagnose/SKILL.md +334 -11
- data/plugin/skills/woods-investigate/SKILL.md +11 -0
- data/plugin/skills/woods-mcp-config/SKILL.md +79 -8
- data/plugin/skills/woods-setup/SKILL.md +53 -6
- metadata +32 -5
|
@@ -8,6 +8,7 @@ require_relative 'client'
|
|
|
8
8
|
require_relative 'rate_limiter'
|
|
9
9
|
require_relative 'document_builder'
|
|
10
10
|
require_relative 'sync_manifest'
|
|
11
|
+
require_relative 'uri_migration'
|
|
11
12
|
require_relative '../export/typed_reader'
|
|
12
13
|
|
|
13
14
|
module Woods
|
|
@@ -20,7 +21,7 @@ module Woods
|
|
|
20
21
|
# remote document_id of everything last pushed, so each run only PUTs
|
|
21
22
|
# new/changed documents, skips unchanged ones, and deletes documents whose
|
|
22
23
|
# source unit has disappeared. Documents are upserted by URI, so a missing
|
|
23
|
-
# manifest
|
|
24
|
+
# manifest rebuilds current receipts without adopting remote deletion rights.
|
|
24
25
|
#
|
|
25
26
|
# @example
|
|
26
27
|
# exporter = Exporter.new(index_dir: "tmp/woods")
|
|
@@ -61,7 +62,8 @@ module Woods
|
|
|
61
62
|
# @param output [IO] Progress output stream (default: $stdout)
|
|
62
63
|
# @raise [ConfigurationError] if required config is missing
|
|
63
64
|
def initialize(index_dir:, config: Woods.configuration, client: nil, reader: nil,
|
|
64
|
-
manifest: nil, force_full: false, force_purge: false, output: $stdout
|
|
65
|
+
manifest: nil, force_full: false, force_purge: false, output: $stdout,
|
|
66
|
+
migrate_from_ref: nil, dry_run: false)
|
|
65
67
|
@collection_id = config.unblocked_collection_id
|
|
66
68
|
raise ConfigurationError, 'unblocked_collection_id is required' unless @collection_id
|
|
67
69
|
|
|
@@ -80,7 +82,9 @@ module Woods
|
|
|
80
82
|
# Cite the ref the index was actually extracted from. `main` was
|
|
81
83
|
# hardcoded, so citations on a `master`-default repo pointed at a
|
|
82
84
|
# branch that need not exist.
|
|
83
|
-
@
|
|
85
|
+
@repo_url = repo_url.chomp('/')
|
|
86
|
+
@migrate_from_ref = migrate_from_ref
|
|
87
|
+
@dry_run = dry_run
|
|
84
88
|
@manifest = manifest || build_manifest(index_dir)
|
|
85
89
|
@force_full = force_full
|
|
86
90
|
@force_purge = force_purge
|
|
@@ -99,11 +103,13 @@ module Woods
|
|
|
99
103
|
# @return [Hash] { synced:, skipped:, deleted:, errors: }
|
|
100
104
|
def sync_all
|
|
101
105
|
prepared = false
|
|
106
|
+
@stats = nil
|
|
102
107
|
with_prepared_index do
|
|
103
108
|
prepared = true
|
|
104
109
|
@current_uris = Set.new
|
|
110
|
+
@attempted_uris = Set.new
|
|
105
111
|
@budget_exhausted = false
|
|
106
|
-
|
|
112
|
+
return preview if @dry_run
|
|
107
113
|
|
|
108
114
|
synced = 0
|
|
109
115
|
skipped = 0
|
|
@@ -128,11 +134,32 @@ module Woods
|
|
|
128
134
|
errors.concat(result[:errors])
|
|
129
135
|
end
|
|
130
136
|
|
|
131
|
-
|
|
132
|
-
|
|
137
|
+
unless @budget_exhausted || errors.any?
|
|
138
|
+
pending = @migration.plan.map { |move| move['new_uri'] }.compact.to_set - @attempted_uris
|
|
139
|
+
replacements = @published_units.select { |unit| unit['file_path'] && pending.include?(effective_uri(unit)) }
|
|
140
|
+
result = sync_unit_data(replacements.map { |unit| [unit, unit] })
|
|
141
|
+
synced += result[:synced]
|
|
142
|
+
skipped += result[:skipped]
|
|
143
|
+
errors.concat(result[:errors])
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
deleted = 0
|
|
147
|
+
unless @budget_exhausted || @ambiguous_uris.any? || errors.any?
|
|
148
|
+
deleted = purge_stale(errors)
|
|
149
|
+
deleted += @migration.cleanup(current_uris: @current_uris, errors: errors,
|
|
150
|
+
replacement_hashes: migration_hashes, remote_documents: @remote_documents,
|
|
151
|
+
collection_id: @collection_id)
|
|
152
|
+
end
|
|
153
|
+
if @manifest.unresolved_legacy_ownership?
|
|
154
|
+
errors << 'cleanup incomplete: legacy owned documents remain; select their source ref with ' \
|
|
155
|
+
'UNBLOCKED_MIGRATE_FROM_REF, or review obsolete remote documents and the legacy manifest ' \
|
|
156
|
+
'receipts for manual cleanup'
|
|
157
|
+
end
|
|
158
|
+
@stats = { synced: synced, skipped: skipped, deleted: deleted, errors: cap_errors(errors),
|
|
159
|
+
complete: errors.empty? && !@budget_exhausted && @migration.plan.empty? }
|
|
133
160
|
end
|
|
134
161
|
ensure
|
|
135
|
-
save_manifest if prepared
|
|
162
|
+
save_manifest if prepared && !@dry_run
|
|
136
163
|
end
|
|
137
164
|
|
|
138
165
|
# Sync all units of a given type.
|
|
@@ -141,6 +168,8 @@ module Woods
|
|
|
141
168
|
# @return [Hash] { synced:, skipped:, errors: }
|
|
142
169
|
def sync_type(type)
|
|
143
170
|
with_prepared_index do
|
|
171
|
+
return preview if @dry_run
|
|
172
|
+
|
|
144
173
|
units = units_for(type)
|
|
145
174
|
log " #{type}: #{units.size} units"
|
|
146
175
|
|
|
@@ -155,6 +184,8 @@ module Woods
|
|
|
155
184
|
# @return [Hash] { synced:, skipped:, errors: }
|
|
156
185
|
def sync_type_partial(type, max_count)
|
|
157
186
|
with_prepared_index do
|
|
187
|
+
return preview if @dry_run
|
|
188
|
+
|
|
158
189
|
units = units_for(type)
|
|
159
190
|
units.each { |unit| track_uri(unit) }
|
|
160
191
|
top = units.sort_by { |unit| -(unit['dependents'] || []).size }.first(max_count)
|
|
@@ -173,8 +204,22 @@ module Woods
|
|
|
173
204
|
return yield if @prepared_index
|
|
174
205
|
|
|
175
206
|
with_pinned_index do
|
|
207
|
+
@builder = DocumentBuilder.new(repo_url: @repo_url, ref: extracted_ref)
|
|
176
208
|
@published_units = @typed_reader.all
|
|
177
209
|
build_uri_index
|
|
210
|
+
@manifest.activate_scope(repo_url: @repo_url, ref: @builder.ref,
|
|
211
|
+
current_uris: @published_units.filter_map do |unit|
|
|
212
|
+
effective_uri(unit) if unit['file_path']
|
|
213
|
+
end)
|
|
214
|
+
@migration = UriMigration.new(manifest: @manifest, client: @client, from_ref: @migrate_from_ref,
|
|
215
|
+
replacements: migration_replacements)
|
|
216
|
+
unless @dry_run
|
|
217
|
+
inventory = @client.all_documents(collection_id: nil)
|
|
218
|
+
@remote_documents = inventory.each_with_object({}) do |doc, documents|
|
|
219
|
+
uri = doc['uri']
|
|
220
|
+
documents[uri] = doc if uri.is_a?(String) && !uri.empty?
|
|
221
|
+
end
|
|
222
|
+
end
|
|
178
223
|
@prepared_index = true
|
|
179
224
|
begin
|
|
180
225
|
yield
|
|
@@ -217,6 +262,7 @@ module Woods
|
|
|
217
262
|
|
|
218
263
|
entries_with_data.each do |entry, unit_data|
|
|
219
264
|
track_uri(unit_data)
|
|
265
|
+
(@attempted_uris ||= Set.new).add(effective_uri(unit_data)) if unit_data['file_path']
|
|
220
266
|
if push_document(unit_data) == :skipped
|
|
221
267
|
skipped += 1
|
|
222
268
|
else
|
|
@@ -252,6 +298,10 @@ module Woods
|
|
|
252
298
|
# keeps it, the rest get a `?unit=` suffix so each is a distinct remote
|
|
253
299
|
# document (and a distinct manifest key).
|
|
254
300
|
uri = effective_uri(unit_data)
|
|
301
|
+
remote = @remote_documents && @remote_documents[uri]
|
|
302
|
+
if remote && remote['collectionId'] != @collection_id
|
|
303
|
+
raise Woods::ExtractionError, 'export URI belongs to a different remote collection — push refused'
|
|
304
|
+
end
|
|
255
305
|
|
|
256
306
|
doc = @builder.build(unit_data)
|
|
257
307
|
# An empty body means the credential scrub failed closed (the builders
|
|
@@ -262,7 +312,8 @@ module Woods
|
|
|
262
312
|
end
|
|
263
313
|
|
|
264
314
|
hash = fingerprint(doc)
|
|
265
|
-
|
|
315
|
+
remote_matches = remote && remote['id'] && remote['id'] == @manifest.document_id_for(uri)
|
|
316
|
+
return :skipped if !@force_full && remote_matches && @manifest.unchanged?(uri, hash)
|
|
266
317
|
|
|
267
318
|
response = @client.put_document(
|
|
268
319
|
collection_id: @collection_id,
|
|
@@ -270,7 +321,8 @@ module Woods
|
|
|
270
321
|
body: doc[:body],
|
|
271
322
|
uri: uri
|
|
272
323
|
)
|
|
273
|
-
document_id = (response['id'] if response.is_a?(Hash)) ||
|
|
324
|
+
document_id = (response['id'] if response.is_a?(Hash)) || remote&.fetch('id', nil)
|
|
325
|
+
@remote_documents[uri] = { 'uri' => uri, 'id' => document_id, 'collectionId' => @collection_id }
|
|
274
326
|
@manifest.record(uri: uri, hash: hash, document_id: document_id)
|
|
275
327
|
:synced
|
|
276
328
|
end
|
|
@@ -284,14 +336,31 @@ module Woods
|
|
|
284
336
|
def purge_stale(errors)
|
|
285
337
|
stale = @manifest.stale_uris(@current_uris)
|
|
286
338
|
return 0 if stale.empty?
|
|
287
|
-
|
|
339
|
+
|
|
340
|
+
if guard_blocks_purge?(stale)
|
|
341
|
+
errors << 'cleanup incomplete: mass-deletion guard refused stale documents'
|
|
342
|
+
return 0
|
|
343
|
+
end
|
|
288
344
|
|
|
289
345
|
resolve_missing_document_ids(stale)
|
|
290
346
|
|
|
291
347
|
deleted = 0
|
|
292
348
|
stale.each do |uri|
|
|
349
|
+
remote = @remote_documents[uri]
|
|
350
|
+
unless remote
|
|
351
|
+
@manifest.forget(uri)
|
|
352
|
+
next
|
|
353
|
+
end
|
|
293
354
|
document_id = @manifest.document_id_for(uri)
|
|
294
|
-
|
|
355
|
+
unless document_id
|
|
356
|
+
errors << "cleanup incomplete: remote document ID unresolved for #{uri}"
|
|
357
|
+
next
|
|
358
|
+
end
|
|
359
|
+
|
|
360
|
+
unless remote['id'] == document_id && remote['collectionId'] == @collection_id
|
|
361
|
+
errors << "cleanup incomplete: remote identity/collection changed for #{uri}"
|
|
362
|
+
next
|
|
363
|
+
end
|
|
295
364
|
|
|
296
365
|
@client.delete_document(document_id: document_id)
|
|
297
366
|
@manifest.forget(uri)
|
|
@@ -324,8 +393,8 @@ module Woods
|
|
|
324
393
|
missing = stale.select { |uri| @manifest.document_id_for(uri).nil? }
|
|
325
394
|
return if missing.empty?
|
|
326
395
|
|
|
327
|
-
ids_by_uri = @
|
|
328
|
-
|
|
396
|
+
ids_by_uri = @remote_documents.values.select { |doc| doc['collectionId'] == @collection_id }
|
|
397
|
+
.to_h { |doc| [doc['uri'], doc['id']] }
|
|
329
398
|
missing.each do |uri|
|
|
330
399
|
id = ids_by_uri[uri]
|
|
331
400
|
@manifest.record(uri: uri, hash: nil, document_id: id) if id
|
|
@@ -352,27 +421,47 @@ module Woods
|
|
|
352
421
|
true
|
|
353
422
|
end
|
|
354
423
|
|
|
355
|
-
#
|
|
356
|
-
#
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
@
|
|
424
|
+
# A preview reads published data and local ownership only. It makes no
|
|
425
|
+
# remote requests and never persists the in-memory migration plan.
|
|
426
|
+
def preview
|
|
427
|
+
{ synced: 0, skipped: 0, deleted: 0, errors: [], complete: false, dry_run: true,
|
|
428
|
+
migration: @migration.plan, scope: { repo_url: @repo_url, ref: @builder.ref } }
|
|
429
|
+
end
|
|
430
|
+
|
|
431
|
+
def migration_replacements
|
|
432
|
+
return {} unless @migrate_from_ref
|
|
433
|
+
|
|
434
|
+
legacy_builder = DocumentBuilder.new(repo_url: @repo_url, ref: @migrate_from_ref)
|
|
435
|
+
@published_units.each_with_object({}) do |unit, mapping|
|
|
436
|
+
next unless unit['file_path']
|
|
437
|
+
next if @ambiguous_uris.include?(@builder.uri_for(unit))
|
|
438
|
+
|
|
439
|
+
old_base = legacy_builder.uri_for(unit)
|
|
440
|
+
old_raw = old_base.sub("/blob/#{encode_ref(@migrate_from_ref)}/", "/blob/#{@migrate_from_ref}/")
|
|
441
|
+
suffix = effective_uri(unit).delete_prefix(@builder.uri_for(unit))
|
|
442
|
+
# A v1 bare URI has no typed owner receipt. A new sibling may now
|
|
443
|
+
# own that bare URI; do not guess which historical unit it replaced.
|
|
444
|
+
next if suffix.empty? && @uri_primary.key?(@builder.uri_for(unit))
|
|
445
|
+
|
|
446
|
+
[old_base, old_raw].uniq.each { |base| mapping[base + suffix] = effective_uri(unit) }
|
|
369
447
|
end
|
|
370
|
-
|
|
371
|
-
raise if [401, 403].include?(e.status)
|
|
448
|
+
end
|
|
372
449
|
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
450
|
+
def migration_hashes
|
|
451
|
+
pending = @migration.plan.map { |move| move['new_uri'] }.compact.to_set
|
|
452
|
+
@published_units.each_with_object({}) do |unit, hashes|
|
|
453
|
+
next unless unit['file_path']
|
|
454
|
+
|
|
455
|
+
uri = effective_uri(unit)
|
|
456
|
+
next unless pending.include?(uri)
|
|
457
|
+
|
|
458
|
+
document = @builder.build(unit)
|
|
459
|
+
hashes[uri] = fingerprint(document) unless document[:body].to_s.empty?
|
|
460
|
+
end
|
|
461
|
+
end
|
|
462
|
+
|
|
463
|
+
def encode_ref(ref)
|
|
464
|
+
ref.split('/').map { |segment| ERB::Util.url_encode(segment) }.join('/')
|
|
376
465
|
end
|
|
377
466
|
|
|
378
467
|
def track_uri(unit_data)
|
|
@@ -457,14 +546,15 @@ module Woods
|
|
|
457
546
|
nil
|
|
458
547
|
end
|
|
459
548
|
|
|
460
|
-
# Persist
|
|
461
|
-
#
|
|
462
|
-
# otherwise-successful sync into a crash (this runs from an ensure, where
|
|
463
|
-
# a raise would also mask any in-flight exception).
|
|
549
|
+
# Persist receipts without masking an in-flight exception. A failed save
|
|
550
|
+
# leaves completion false: cleanup requires durable ownership evidence.
|
|
464
551
|
def save_manifest
|
|
465
552
|
@manifest.save
|
|
466
553
|
rescue StandardError => e
|
|
467
|
-
|
|
554
|
+
message = "sync manifest not persisted: #{e.class}: #{e.message}"
|
|
555
|
+
@stats[:errors] << message if @stats
|
|
556
|
+
@stats[:complete] = false if @stats
|
|
557
|
+
log " WARNING: #{message} — recover local ownership state before cleanup"
|
|
468
558
|
end
|
|
469
559
|
|
|
470
560
|
def build_manifest(index_dir)
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
require 'json'
|
|
4
4
|
require 'fileutils'
|
|
5
|
+
require 'digest'
|
|
6
|
+
require 'erb'
|
|
5
7
|
|
|
6
8
|
require_relative '../atomic_file'
|
|
7
9
|
|
|
@@ -14,8 +16,8 @@ module Woods
|
|
|
14
16
|
# entry records the content hash of the document we last pushed for a URI
|
|
15
17
|
# plus the remote +document_id+ (needed for deletes). Persisted as JSON
|
|
16
18
|
# alongside the extraction output and restored across CI runs via the CI
|
|
17
|
-
# provider's cache.
|
|
18
|
-
#
|
|
19
|
+
# provider's cache. Missing receipts never establish remote deletion rights.
|
|
20
|
+
# Corrupt or mismatched manifests require recovery before another sync.
|
|
19
21
|
#
|
|
20
22
|
# Modeled on the embedding indexer's checkpoint (load JSON → compare
|
|
21
23
|
# per-key hash → save JSON).
|
|
@@ -28,7 +30,7 @@ module Woods
|
|
|
28
30
|
# manifest.save
|
|
29
31
|
#
|
|
30
32
|
class SyncManifest
|
|
31
|
-
VERSION =
|
|
33
|
+
VERSION = 2
|
|
32
34
|
|
|
33
35
|
# @param path [String] JSON file path for the manifest
|
|
34
36
|
# @param collection_id [String] Target collection UUID — a stored manifest
|
|
@@ -36,7 +38,10 @@ module Woods
|
|
|
36
38
|
def initialize(path:, collection_id:)
|
|
37
39
|
@path = path
|
|
38
40
|
@collection_id = collection_id
|
|
41
|
+
@scopes = {}
|
|
39
42
|
@documents = load
|
|
43
|
+
@legacy_documents = @documents
|
|
44
|
+
@documents = @scopes.dig(@active_scope, 'documents') || @legacy_documents
|
|
40
45
|
end
|
|
41
46
|
|
|
42
47
|
# @return [Boolean] true when no documents are recorded
|
|
@@ -58,7 +63,8 @@ module Woods
|
|
|
58
63
|
# @param hash [String, nil] Content hash pushed (nil forces a future re-push)
|
|
59
64
|
# @param document_id [String, nil] Remote document UUID (for later deletes)
|
|
60
65
|
def record(uri:, hash:, document_id:)
|
|
61
|
-
|
|
66
|
+
owned = !hash.nil? || @documents.dig(uri, 'owned') == true
|
|
67
|
+
@documents[uri] = { 'hash' => hash, 'document_id' => document_id, 'owned' => owned }
|
|
62
68
|
end
|
|
63
69
|
|
|
64
70
|
# @param uri [String] Document URI
|
|
@@ -94,38 +100,145 @@ module Woods
|
|
|
94
100
|
payload = JSON.generate(
|
|
95
101
|
'version' => VERSION,
|
|
96
102
|
'collection_id' => @collection_id,
|
|
97
|
-
'documents' => @
|
|
103
|
+
'documents' => @legacy_documents,
|
|
104
|
+
'scopes' => @scopes,
|
|
105
|
+
'active_scope' => @active_scope
|
|
98
106
|
)
|
|
99
107
|
AtomicFile.write(@path, payload)
|
|
100
108
|
end
|
|
101
109
|
|
|
110
|
+
# Select one repository/ref without retiring any other branch's documents.
|
|
111
|
+
# Existing v1 entries are adopted only when they prove a successful write
|
|
112
|
+
# and correspond to a currently published URI in this exact scope. A URI
|
|
113
|
+
# prefix cannot distinguish refs containing slashes from source paths.
|
|
114
|
+
def activate_scope(repo_url:, ref:, current_uris:)
|
|
115
|
+
raise Woods::ConfigurationError, "Unblocked manifest needs recovery: #{@load_error}" if @load_error
|
|
116
|
+
|
|
117
|
+
@repo_url = repo_url.chomp('/')
|
|
118
|
+
@ref = ref
|
|
119
|
+
@active_scope = scope_key(@repo_url, ref)
|
|
120
|
+
@scopes[@active_scope] ||= { 'repo_url' => @repo_url, 'ref' => ref, 'documents' => {} }
|
|
121
|
+
@documents = @scopes.fetch(@active_scope).fetch('documents')
|
|
122
|
+
current_uris.each do |uri|
|
|
123
|
+
entry = @legacy_documents[uri]
|
|
124
|
+
next unless entry && owned?(entry)
|
|
125
|
+
|
|
126
|
+
@documents[uri] ||= entry
|
|
127
|
+
@legacy_documents.delete(uri)
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# A remaining v1 receipt has no selected repository/ref scope. Report
|
|
132
|
+
# the cleanup obligation rather than claiming that synchronization is done.
|
|
133
|
+
def unresolved_legacy_ownership?
|
|
134
|
+
@legacy_documents.any? { |uri, entry| uri.start_with?("#{@repo_url}/blob/") && owned?(entry) }
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Local ownership evidence for an explicitly selected migration source.
|
|
138
|
+
# A remote listing never establishes permission to delete a document.
|
|
139
|
+
def migration_sources(from_ref:)
|
|
140
|
+
key = scope_key(@repo_url, from_ref)
|
|
141
|
+
scoped = @scopes.dig(key, 'documents') || {}
|
|
142
|
+
prefixes = [from_ref, encode_ref(from_ref)].uniq.map { |ref| "#{@repo_url}/blob/#{ref}/" }
|
|
143
|
+
legacy = @legacy_documents.select { |uri, _| prefixes.any? { |prefix| uri.start_with?(prefix) } }
|
|
144
|
+
[[:legacy, legacy], [key, scoped]].flat_map do |location, entries|
|
|
145
|
+
entries.map do |uri, entry|
|
|
146
|
+
entry.merge('old_uri' => uri, 'source_scope' => location.to_s, 'owned' => owned?(entry))
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Persist the approved source scope and bounded set of replacement pairs.
|
|
152
|
+
def start_migration(from_ref:, moves:)
|
|
153
|
+
finish_migration
|
|
154
|
+
current = migration
|
|
155
|
+
if current && current['from_ref'] != from_ref
|
|
156
|
+
raise Woods::ConfigurationError, 'Finish the pending Unblocked migration before selecting another source ref'
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
return if current.nil? && moves.empty?
|
|
160
|
+
|
|
161
|
+
@scopes.fetch(@active_scope)['migration'] ||= { 'from_ref' => from_ref, 'moves' => moves }
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def migration
|
|
165
|
+
@scopes.dig(@active_scope, 'migration')
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def complete_move(move)
|
|
169
|
+
source = if move.fetch('source_scope') == 'legacy'
|
|
170
|
+
@legacy_documents
|
|
171
|
+
else
|
|
172
|
+
@scopes.dig(move.fetch('source_scope'), 'documents')
|
|
173
|
+
end
|
|
174
|
+
source&.delete(move.fetch('old_uri'))
|
|
175
|
+
migration.fetch('moves').delete(move)
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# Finish only after every selected replacement and old-copy cleanup.
|
|
179
|
+
def finish_migration
|
|
180
|
+
@scopes.fetch(@active_scope).delete('migration') if migration && migration.fetch('moves').empty?
|
|
181
|
+
end
|
|
182
|
+
|
|
102
183
|
private
|
|
103
184
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
185
|
+
def scope_key(repo, ref)
|
|
186
|
+
Digest::SHA256.hexdigest(JSON.generate([repo, ref]))
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def encode_ref(ref)
|
|
190
|
+
ref.split('/').map { |segment| ERB::Util.url_encode(segment) }.join('/')
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def owned?(entry)
|
|
194
|
+
entry['owned'] == true || (entry['hash'].is_a?(String) && !entry['hash'].empty?)
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def valid_documents?(documents)
|
|
198
|
+
documents.is_a?(Hash) && documents.all? { |uri, entry| uri.is_a?(String) && entry.is_a?(Hash) }
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def valid_scopes?(scopes)
|
|
202
|
+
scopes.is_a?(Hash) && scopes.all? do |key, scope|
|
|
203
|
+
key.is_a?(String) && scope.is_a?(Hash) && scope['repo_url'].is_a?(String) &&
|
|
204
|
+
scope['ref'].is_a?(String) && key == scope_key(scope['repo_url'], scope['ref']) &&
|
|
205
|
+
valid_documents?(scope['documents']) && valid_migration?(scope['migration'])
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def valid_migration?(migration)
|
|
210
|
+
return true if migration.nil?
|
|
211
|
+
unless migration.is_a?(Hash) && migration['from_ref'].is_a?(String) && migration['moves'].is_a?(Array)
|
|
212
|
+
return false
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
migration['moves'].all? do |move|
|
|
216
|
+
move.is_a?(Hash) && move['old_uri'].is_a?(String) && move['source_scope'].is_a?(String) &&
|
|
217
|
+
(move['new_uri'].nil? || move['new_uri'].is_a?(String)) && [true, false].include?(move['owned'])
|
|
218
|
+
end
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# Retain legacy records, but refuse malformed ownership state at sync
|
|
222
|
+
# preflight. AtomicFile supplies UTF-8 independently of the host locale.
|
|
116
223
|
# @return [Hash{String=>Hash}] uri => { 'hash' =>, 'document_id' => }
|
|
117
224
|
def load
|
|
118
225
|
return {} unless File.exist?(@path)
|
|
119
226
|
|
|
120
227
|
parsed = JSON.parse(AtomicFile.read(@path))
|
|
121
228
|
return discard('not a JSON object') unless parsed.is_a?(Hash)
|
|
122
|
-
return discard("schema version #{parsed['version'].inspect}, expected #{VERSION}") unless
|
|
123
|
-
parsed['version']
|
|
229
|
+
return discard("schema version #{parsed['version'].inspect}, expected 1 or #{VERSION}") unless
|
|
230
|
+
[1, VERSION].include?(parsed['version'])
|
|
124
231
|
return discard("written for collection #{parsed['collection_id'].inspect}, expected #{@collection_id}") unless
|
|
125
232
|
parsed['collection_id'] == @collection_id
|
|
126
233
|
|
|
234
|
+
@scopes = parsed.fetch('scopes', {})
|
|
235
|
+
return discard('invalid scoped ownership records') unless valid_scopes?(@scopes)
|
|
236
|
+
|
|
237
|
+
@active_scope = parsed['active_scope']
|
|
127
238
|
documents = parsed['documents']
|
|
128
|
-
|
|
239
|
+
return discard('invalid document ownership records') unless valid_documents?(documents)
|
|
240
|
+
|
|
241
|
+
documents
|
|
129
242
|
rescue JSON::ParserError
|
|
130
243
|
discard('unparseable JSON')
|
|
131
244
|
rescue EncodingError, SystemCallError => e
|
|
@@ -135,7 +248,10 @@ module Woods
|
|
|
135
248
|
# @param reason [String] Why the persisted manifest is unusable
|
|
136
249
|
# @return [Hash] empty documents hash (degrades to a full re-push)
|
|
137
250
|
def discard(reason)
|
|
138
|
-
|
|
251
|
+
@load_error = reason
|
|
252
|
+
@scopes = {}
|
|
253
|
+
warn "WARNING: discarding sync manifest at #{@path} (#{reason}) — " \
|
|
254
|
+
'sync requires manifest recovery before remote changes'
|
|
139
255
|
{}
|
|
140
256
|
end
|
|
141
257
|
end
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Woods
|
|
4
|
+
module Unblocked
|
|
5
|
+
# Resumable, explicitly selected URI replacements. Ordinary branch changes
|
|
6
|
+
# never enter this path and never retire another branch's ownership scope.
|
|
7
|
+
class UriMigration
|
|
8
|
+
# @param manifest [SyncManifest] currently selected target scope
|
|
9
|
+
# @param client [Client] remote document operations
|
|
10
|
+
# @param from_ref [String, nil] explicit source branch, or resume pending work
|
|
11
|
+
# @param replacements [Hash{String => String}] old URI to published target URI
|
|
12
|
+
def initialize(manifest:, client:, from_ref:, replacements:)
|
|
13
|
+
@manifest = manifest
|
|
14
|
+
@client = client
|
|
15
|
+
return unless from_ref
|
|
16
|
+
|
|
17
|
+
moves = manifest.migration_sources(from_ref: from_ref).filter_map do |entry|
|
|
18
|
+
target = replacements[entry.fetch('old_uri')]
|
|
19
|
+
next if target == entry.fetch('old_uri')
|
|
20
|
+
|
|
21
|
+
entry.merge('new_uri' => target)
|
|
22
|
+
end
|
|
23
|
+
manifest.start_migration(from_ref: from_ref, moves: moves)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# @return [Array<Hash>] exact pending replacements, suitable for preview
|
|
27
|
+
def plan
|
|
28
|
+
@manifest.migration&.fetch('moves') || []
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Save successful replacement receipts before deleting any old copies.
|
|
32
|
+
# This explicit migration set has its own ownership checks; the ordinary
|
|
33
|
+
# mass-delete ratio does not mistake a one-for-one migration for loss.
|
|
34
|
+
# @param current_uris [Set<String>] verified current publication membership
|
|
35
|
+
# @param errors [Array<String>] actionable refusal/failure messages
|
|
36
|
+
# @return [Integer] old copies removed
|
|
37
|
+
def cleanup(current_uris:, errors:, replacement_hashes:, remote_documents:, collection_id:)
|
|
38
|
+
return 0 if plan.empty?
|
|
39
|
+
|
|
40
|
+
@manifest.save
|
|
41
|
+
deleted = 0
|
|
42
|
+
plan.dup.each do |move|
|
|
43
|
+
remote = remote_documents[move.fetch('old_uri')]
|
|
44
|
+
if remote.nil?
|
|
45
|
+
finish_move(move)
|
|
46
|
+
next
|
|
47
|
+
end
|
|
48
|
+
reason = refusal(move, current_uris, replacement_hashes)
|
|
49
|
+
target = remote_documents[move['new_uri']]
|
|
50
|
+
unless target && target['id'] == @manifest.document_id_for(move['new_uri']) &&
|
|
51
|
+
target['collectionId'] == collection_id
|
|
52
|
+
reason ||= 'replacement remote identity is missing or changed'
|
|
53
|
+
end
|
|
54
|
+
reason ||= 'source remote identity/collection changed; review before cleanup' unless
|
|
55
|
+
remote['id'] == move['document_id'] && remote['collectionId'] == collection_id
|
|
56
|
+
if reason
|
|
57
|
+
errors << "URI migration incomplete: #{reason} (#{move.fetch('old_uri')})"
|
|
58
|
+
next
|
|
59
|
+
end
|
|
60
|
+
@client.delete_document(document_id: move.fetch('document_id'))
|
|
61
|
+
deleted += 1
|
|
62
|
+
finish_move(move)
|
|
63
|
+
rescue ApiError => e
|
|
64
|
+
if e.status == 404
|
|
65
|
+
finish_move(move)
|
|
66
|
+
else
|
|
67
|
+
errors << "URI migration cleanup failed: #{e.message}"
|
|
68
|
+
end
|
|
69
|
+
rescue StandardError => e
|
|
70
|
+
errors << "URI migration cleanup incomplete: #{e.class}: #{e.message}"
|
|
71
|
+
break
|
|
72
|
+
end
|
|
73
|
+
@manifest.finish_migration
|
|
74
|
+
deleted
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
private
|
|
78
|
+
|
|
79
|
+
def refusal(move, current_uris, replacement_hashes)
|
|
80
|
+
unless move['owned'] == true
|
|
81
|
+
return 'legacy ownership is unverified; review the remote record before manual cleanup'
|
|
82
|
+
end
|
|
83
|
+
unless move['document_id'].is_a?(String) && !move['document_id'].empty?
|
|
84
|
+
return 'source document ID is unresolved'
|
|
85
|
+
end
|
|
86
|
+
return 'no unambiguous current replacement' unless current_uris.include?(move['new_uri'])
|
|
87
|
+
|
|
88
|
+
expected = replacement_hashes[move['new_uri']]
|
|
89
|
+
return 'replacement content has not been successfully synchronized' unless
|
|
90
|
+
expected && @manifest.unchanged?(move['new_uri'], expected)
|
|
91
|
+
|
|
92
|
+
target_id = @manifest.document_id_for(move['new_uri'])
|
|
93
|
+
return 'replacement has no persisted successful document receipt' unless target_id
|
|
94
|
+
return 'source and replacement document IDs are identical' if target_id == move['document_id']
|
|
95
|
+
|
|
96
|
+
nil
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def finish_move(move)
|
|
100
|
+
@manifest.complete_move(move)
|
|
101
|
+
@manifest.save
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
module Woods
|
|
4
4
|
module Util
|
|
5
|
-
#
|
|
6
|
-
#
|
|
5
|
+
# Numeric-host checks shared by {MCP::OriginPolicy} and the
|
|
6
|
+
# {Storage::VectorStore::Qdrant} URL validator. Qdrant also uses the URL
|
|
7
|
+
# canonicalization helper; HTTP authorities retain SDK matching semantics.
|
|
7
8
|
#
|
|
8
9
|
# Both components need to reject numeric IPv4 notations that `URI` and
|
|
9
10
|
# `getaddrinfo` accept but `IPAddr` does not — hex (`0x7f000001`),
|
data/lib/woods/version.rb
CHANGED