woods 2.0.0 → 2.1.0

data/lib/woods/unblocked/client.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'json'
+require 'set'
 require 'net/http'
 require 'uri'
 require 'woods'
@@ -145,9 +146,12 @@ module Woods
         query = "limit=#{limit}"
         query += "&after=#{URI.encode_www_form_component(after)}" if after
         result = request(:get, "documents?#{query}")
-        return result if result.is_a?(Array)
+        page = result.is_a?(Array) ? result : result.is_a?(Hash) && (result['items'] || result['data'])
+        unless page.is_a?(Array) && page.all?(Hash)
+          raise Woods::Error, 'Unblocked document inventory incomplete: malformed page'
+        end
 
-        result['items'] || result['data'] || []
+        page
       end
 
       # List every document in a collection, paging until exhausted.
@@ -160,20 +164,21 @@ module Woods
       def all_documents(collection_id:)
         docs = []
         after = nil
+        seen_cursors = Set.new
 
         loop do
           page = list_documents(limit: PAGE_SIZE, after: after)
           break if page.empty?
 
           docs.concat(page)
-          break if page.size < PAGE_SIZE
-
           after = page.last['id']
-          # A full page with no cursor id would refetch page 1 forever —
-          # stop with what we have rather than loop against the budget.
-          break if after.nil?
+          unless after.is_a?(String) && !after.empty? && seen_cursors.add?(after)
+            raise Woods::Error, 'Unblocked document inventory incomplete: missing or repeated pagination cursor'
+          end
         end
 
+        return docs if collection_id.nil?
+
         docs.select { |doc| doc['collectionId'] == collection_id }
       end
 

data/lib/woods/unblocked/document_builder.rb

@@ -28,6 +28,9 @@ module Woods
         @ref = normalize_ref(ref)
       end
 
+      # @return [String] normalized branch/ref used in citation and ownership identity
+      attr_reader :ref
+
       # Ref used when the index does not say which one it came from.
       DEFAULT_REF = 'main'
 
@@ -60,7 +63,7 @@ module Woods
         file_path = unit_data['file_path']
         return @repo_url unless file_path
 
-        "#{@repo_url}/blob/#{@ref}/#{encode_path(file_path)}"
+        "#{@repo_url}/blob/#{encode_path(@ref)}/#{encode_path(file_path)}"
       end
 
       private

data/lib/woods/unblocked/exporter.rb

@@ -8,6 +8,7 @@ require_relative 'client'
 require_relative 'rate_limiter'
 require_relative 'document_builder'
 require_relative 'sync_manifest'
+require_relative 'uri_migration'
 require_relative '../export/typed_reader'
 
 module Woods
@@ -20,7 +21,7 @@ module Woods
     # remote document_id of everything last pushed, so each run only PUTs
     # new/changed documents, skips unchanged ones, and deletes documents whose
     # source unit has disappeared. Documents are upserted by URI, so a missing
-    # manifest (first run / CI cache miss) degrades to a correct full sync.
+    # manifest rebuilds current receipts without adopting remote deletion rights.
     #
     # @example
     #   exporter = Exporter.new(index_dir: "tmp/woods")
@@ -61,7 +62,8 @@ module Woods
       # @param output [IO] Progress output stream (default: $stdout)
       # @raise [ConfigurationError] if required config is missing
       def initialize(index_dir:, config: Woods.configuration, client: nil, reader: nil,
-                     manifest: nil, force_full: false, force_purge: false, output: $stdout)
+                     manifest: nil, force_full: false, force_purge: false, output: $stdout,
+                     migrate_from_ref: nil, dry_run: false)
         @collection_id = config.unblocked_collection_id
         raise ConfigurationError, 'unblocked_collection_id is required' unless @collection_id
 
@@ -80,7 +82,9 @@ module Woods
         # Cite the ref the index was actually extracted from. `main` was
         # hardcoded, so citations on a `master`-default repo pointed at a
         # branch that need not exist.
-        @builder = DocumentBuilder.new(repo_url: repo_url, ref: extracted_ref)
+        @repo_url = repo_url.chomp('/')
+        @migrate_from_ref = migrate_from_ref
+        @dry_run = dry_run
         @manifest = manifest || build_manifest(index_dir)
         @force_full = force_full
         @force_purge = force_purge
@@ -99,11 +103,13 @@ module Woods
       # @return [Hash] { synced:, skipped:, deleted:, errors: }
       def sync_all
         prepared = false
+        @stats = nil
         with_prepared_index do
           prepared = true
           @current_uris = Set.new
+          @attempted_uris = Set.new
           @budget_exhausted = false
-          reconcile_from_remote if @manifest.empty?
+          return preview if @dry_run
 
           synced = 0
           skipped = 0
@@ -128,11 +134,32 @@ module Woods
             errors.concat(result[:errors])
           end
 
-          deleted = @budget_exhausted || @ambiguous_uris.any? ? 0 : purge_stale(errors)
-          { synced: synced, skipped: skipped, deleted: deleted, errors: cap_errors(errors) }
+          unless @budget_exhausted || errors.any?
+            pending = @migration.plan.map { |move| move['new_uri'] }.compact.to_set - @attempted_uris
+            replacements = @published_units.select { |unit| unit['file_path'] && pending.include?(effective_uri(unit)) }
+            result = sync_unit_data(replacements.map { |unit| [unit, unit] })
+            synced += result[:synced]
+            skipped += result[:skipped]
+            errors.concat(result[:errors])
+          end
+
+          deleted = 0
+          unless @budget_exhausted || @ambiguous_uris.any? || errors.any?
+            deleted = purge_stale(errors)
+            deleted += @migration.cleanup(current_uris: @current_uris, errors: errors,
+                                          replacement_hashes: migration_hashes, remote_documents: @remote_documents,
+                                          collection_id: @collection_id)
+          end
+          if @manifest.unresolved_legacy_ownership?
+            errors << 'cleanup incomplete: legacy owned documents remain; select their source ref with ' \
+                      'UNBLOCKED_MIGRATE_FROM_REF, or review obsolete remote documents and the legacy manifest ' \
+                      'receipts for manual cleanup'
+          end
+          @stats = { synced: synced, skipped: skipped, deleted: deleted, errors: cap_errors(errors),
+                     complete: errors.empty? && !@budget_exhausted && @migration.plan.empty? }
         end
       ensure
-        save_manifest if prepared
+        save_manifest if prepared && !@dry_run
       end
 
       # Sync all units of a given type.
@@ -141,6 +168,8 @@ module Woods
       # @return [Hash] { synced:, skipped:, errors: }
       def sync_type(type)
         with_prepared_index do
+          return preview if @dry_run
+
           units = units_for(type)
           log "  #{type}: #{units.size} units"
 
@@ -155,6 +184,8 @@ module Woods
       # @return [Hash] { synced:, skipped:, errors: }
       def sync_type_partial(type, max_count)
         with_prepared_index do
+          return preview if @dry_run
+
           units = units_for(type)
           units.each { |unit| track_uri(unit) }
           top = units.sort_by { |unit| -(unit['dependents'] || []).size }.first(max_count)
@@ -173,8 +204,22 @@ module Woods
         return yield if @prepared_index
 
         with_pinned_index do
+          @builder = DocumentBuilder.new(repo_url: @repo_url, ref: extracted_ref)
           @published_units = @typed_reader.all
           build_uri_index
+          @manifest.activate_scope(repo_url: @repo_url, ref: @builder.ref,
+                                   current_uris: @published_units.filter_map do |unit|
+                                     effective_uri(unit) if unit['file_path']
+                                   end)
+          @migration = UriMigration.new(manifest: @manifest, client: @client, from_ref: @migrate_from_ref,
+                                        replacements: migration_replacements)
+          unless @dry_run
+            inventory = @client.all_documents(collection_id: nil)
+            @remote_documents = inventory.each_with_object({}) do |doc, documents|
+              uri = doc['uri']
+              documents[uri] = doc if uri.is_a?(String) && !uri.empty?
+            end
+          end
           @prepared_index = true
           begin
             yield
@@ -217,6 +262,7 @@ module Woods
 
         entries_with_data.each do |entry, unit_data|
           track_uri(unit_data)
+          (@attempted_uris ||= Set.new).add(effective_uri(unit_data)) if unit_data['file_path']
           if push_document(unit_data) == :skipped
             skipped += 1
           else
@@ -252,6 +298,10 @@ module Woods
         # keeps it, the rest get a `?unit=` suffix so each is a distinct remote
         # document (and a distinct manifest key).
         uri = effective_uri(unit_data)
+        remote = @remote_documents && @remote_documents[uri]
+        if remote && remote['collectionId'] != @collection_id
+          raise Woods::ExtractionError, 'export URI belongs to a different remote collection — push refused'
+        end
 
         doc = @builder.build(unit_data)
         # An empty body means the credential scrub failed closed (the builders
@@ -262,7 +312,8 @@ module Woods
         end
 
         hash = fingerprint(doc)
-        return :skipped if !@force_full && @manifest.unchanged?(uri, hash)
+        remote_matches = remote && remote['id'] && remote['id'] == @manifest.document_id_for(uri)
+        return :skipped if !@force_full && remote_matches && @manifest.unchanged?(uri, hash)
 
         response = @client.put_document(
           collection_id: @collection_id,
@@ -270,7 +321,8 @@ module Woods
           body: doc[:body],
           uri: uri
         )
-        document_id = (response['id'] if response.is_a?(Hash)) || @manifest.document_id_for(uri)
+        document_id = (response['id'] if response.is_a?(Hash)) || remote&.fetch('id', nil)
+        @remote_documents[uri] = { 'uri' => uri, 'id' => document_id, 'collectionId' => @collection_id }
         @manifest.record(uri: uri, hash: hash, document_id: document_id)
         :synced
       end
@@ -284,14 +336,31 @@ module Woods
       def purge_stale(errors)
         stale = @manifest.stale_uris(@current_uris)
         return 0 if stale.empty?
-        return 0 if guard_blocks_purge?(stale)
+
+        if guard_blocks_purge?(stale)
+          errors << 'cleanup incomplete: mass-deletion guard refused stale documents'
+          return 0
+        end
 
         resolve_missing_document_ids(stale)
 
         deleted = 0
         stale.each do |uri|
+          remote = @remote_documents[uri]
+          unless remote
+            @manifest.forget(uri)
+            next
+          end
           document_id = @manifest.document_id_for(uri)
-          next unless document_id
+          unless document_id
+            errors << "cleanup incomplete: remote document ID unresolved for #{uri}"
+            next
+          end
+
+          unless remote['id'] == document_id && remote['collectionId'] == @collection_id
+            errors << "cleanup incomplete: remote identity/collection changed for #{uri}"
+            next
+          end
 
           @client.delete_document(document_id: document_id)
           @manifest.forget(uri)
@@ -324,8 +393,8 @@ module Woods
         missing = stale.select { |uri| @manifest.document_id_for(uri).nil? }
         return if missing.empty?
 
-        ids_by_uri = @client.all_documents(collection_id: @collection_id)
-                            .to_h { |doc| [doc['uri'], doc['id']] }
+        ids_by_uri = @remote_documents.values.select { |doc| doc['collectionId'] == @collection_id }
+                                      .to_h { |doc| [doc['uri'], doc['id']] }
         missing.each do |uri|
           id = ids_by_uri[uri]
           @manifest.record(uri: uri, hash: nil, document_id: id) if id
@@ -352,27 +421,47 @@ module Woods
         true
       end
 
-      # Seed the manifest from the remote collection when we have no local
-      # state (first run / CI cache miss). The list endpoint returns no body,
-      # so hashes are nil (everything re-pushes), but recovering document_ids
-      # lets this run still purge orphaned documents.
-      #
-      # Auth failures re-raise: a 401/403 here dooms every subsequent call,
-      # and "proceeding with full sync" would burn the whole daily budget on
-      # guaranteed failures.
-      def reconcile_from_remote
-        @client.all_documents(collection_id: @collection_id).each do |doc|
-          uri = doc['uri']
-          next unless uri
-
-          @manifest.record(uri: uri, hash: nil, document_id: doc['id'])
+      # A preview reads published data and local ownership only. It makes no
+      # remote requests and never persists the in-memory migration plan.
+      def preview
+        { synced: 0, skipped: 0, deleted: 0, errors: [], complete: false, dry_run: true,
+          migration: @migration.plan, scope: { repo_url: @repo_url, ref: @builder.ref } }
+      end
+
+      def migration_replacements
+        return {} unless @migrate_from_ref
+
+        legacy_builder = DocumentBuilder.new(repo_url: @repo_url, ref: @migrate_from_ref)
+        @published_units.each_with_object({}) do |unit, mapping|
+          next unless unit['file_path']
+          next if @ambiguous_uris.include?(@builder.uri_for(unit))
+
+          old_base = legacy_builder.uri_for(unit)
+          old_raw = old_base.sub("/blob/#{encode_ref(@migrate_from_ref)}/", "/blob/#{@migrate_from_ref}/")
+          suffix = effective_uri(unit).delete_prefix(@builder.uri_for(unit))
+          # A v1 bare URI has no typed owner receipt. A new sibling may now
+          # own that bare URI; do not guess which historical unit it replaced.
+          next if suffix.empty? && @uri_primary.key?(@builder.uri_for(unit))
+
+          [old_base, old_raw].uniq.each { |base| mapping[base + suffix] = effective_uri(unit) }
         end
-      rescue ApiError => e
-        raise if [401, 403].include?(e.status)
+      end
 
-        log "  reconcile skipped (#{e.message}) — proceeding with full sync"
-      rescue StandardError => e
-        log "  reconcile skipped (#{e.message}) — proceeding with full sync"
+      def migration_hashes
+        pending = @migration.plan.map { |move| move['new_uri'] }.compact.to_set
+        @published_units.each_with_object({}) do |unit, hashes|
+          next unless unit['file_path']
+
+          uri = effective_uri(unit)
+          next unless pending.include?(uri)
+
+          document = @builder.build(unit)
+          hashes[uri] = fingerprint(document) unless document[:body].to_s.empty?
+        end
+      end
+
+      def encode_ref(ref)
+        ref.split('/').map { |segment| ERB::Util.url_encode(segment) }.join('/')
       end
 
       def track_uri(unit_data)
@@ -457,14 +546,15 @@ module Woods
         nil
       end
 
-      # Persist the manifest, downgrading failures to a warning: losing the
-      # manifest only costs a full re-check next run, which must not turn an
-      # otherwise-successful sync into a crash (this runs from an ensure, where
-      # a raise would also mask any in-flight exception).
+      # Persist receipts without masking an in-flight exception. A failed save
+      # leaves completion false: cleanup requires durable ownership evidence.
       def save_manifest
         @manifest.save
       rescue StandardError => e
-        log "  WARNING: sync manifest not persisted (#{e.message}) — next run will re-push all documents"
+        message = "sync manifest not persisted: #{e.class}: #{e.message}"
+        @stats[:errors] << message if @stats
+        @stats[:complete] = false if @stats
+        log "  WARNING: #{message} — recover local ownership state before cleanup"
       end
 
       def build_manifest(index_dir)

data/lib/woods/unblocked/sync_manifest.rb

@@ -2,6 +2,8 @@
 
 require 'json'
 require 'fileutils'
+require 'digest'
+require 'erb'
 
 require_relative '../atomic_file'
 
@@ -14,8 +16,8 @@ module Woods
     # entry records the content hash of the document we last pushed for a URI
     # plus the remote +document_id+ (needed for deletes). Persisted as JSON
     # alongside the extraction output and restored across CI runs via the CI
-    # provider's cache. A missing or corrupt file degrades to "everything is
-    # new" — a correct (if expensive) full sync that rebuilds the manifest.
+    # provider's cache. Missing receipts never establish remote deletion rights.
+    # Corrupt or mismatched manifests require recovery before another sync.
     #
     # Modeled on the embedding indexer's checkpoint (load JSON → compare
     # per-key hash → save JSON).
@@ -28,7 +30,7 @@ module Woods
     #   manifest.save
     #
     class SyncManifest
-      VERSION = 1
+      VERSION = 2
 
       # @param path [String] JSON file path for the manifest
       # @param collection_id [String] Target collection UUID — a stored manifest
@@ -36,7 +38,10 @@ module Woods
       def initialize(path:, collection_id:)
         @path = path
         @collection_id = collection_id
+        @scopes = {}
         @documents = load
+        @legacy_documents = @documents
+        @documents = @scopes.dig(@active_scope, 'documents') || @legacy_documents
       end
 
       # @return [Boolean] true when no documents are recorded
@@ -58,7 +63,8 @@ module Woods
       # @param hash [String, nil] Content hash pushed (nil forces a future re-push)
       # @param document_id [String, nil] Remote document UUID (for later deletes)
       def record(uri:, hash:, document_id:)
-        @documents[uri] = { 'hash' => hash, 'document_id' => document_id }
+        owned = !hash.nil? || @documents.dig(uri, 'owned') == true
+        @documents[uri] = { 'hash' => hash, 'document_id' => document_id, 'owned' => owned }
       end
 
       # @param uri [String] Document URI
@@ -94,38 +100,145 @@ module Woods
         payload = JSON.generate(
           'version' => VERSION,
           'collection_id' => @collection_id,
-          'documents' => @documents
+          'documents' => @legacy_documents,
+          'scopes' => @scopes,
+          'active_scope' => @active_scope
         )
         AtomicFile.write(@path, payload)
       end
 
+      # Select one repository/ref without retiring any other branch's documents.
+      # Existing v1 entries are adopted only when they prove a successful write
+      # and correspond to a currently published URI in this exact scope. A URI
+      # prefix cannot distinguish refs containing slashes from source paths.
+      def activate_scope(repo_url:, ref:, current_uris:)
+        raise Woods::ConfigurationError, "Unblocked manifest needs recovery: #{@load_error}" if @load_error
+
+        @repo_url = repo_url.chomp('/')
+        @ref = ref
+        @active_scope = scope_key(@repo_url, ref)
+        @scopes[@active_scope] ||= { 'repo_url' => @repo_url, 'ref' => ref, 'documents' => {} }
+        @documents = @scopes.fetch(@active_scope).fetch('documents')
+        current_uris.each do |uri|
+          entry = @legacy_documents[uri]
+          next unless entry && owned?(entry)
+
+          @documents[uri] ||= entry
+          @legacy_documents.delete(uri)
+        end
+      end
+
+      # A remaining v1 receipt has no selected repository/ref scope. Report
+      # the cleanup obligation rather than claiming that synchronization is done.
+      def unresolved_legacy_ownership?
+        @legacy_documents.any? { |uri, entry| uri.start_with?("#{@repo_url}/blob/") && owned?(entry) }
+      end
+
+      # Local ownership evidence for an explicitly selected migration source.
+      # A remote listing never establishes permission to delete a document.
+      def migration_sources(from_ref:)
+        key = scope_key(@repo_url, from_ref)
+        scoped = @scopes.dig(key, 'documents') || {}
+        prefixes = [from_ref, encode_ref(from_ref)].uniq.map { |ref| "#{@repo_url}/blob/#{ref}/" }
+        legacy = @legacy_documents.select { |uri, _| prefixes.any? { |prefix| uri.start_with?(prefix) } }
+        [[:legacy, legacy], [key, scoped]].flat_map do |location, entries|
+          entries.map do |uri, entry|
+            entry.merge('old_uri' => uri, 'source_scope' => location.to_s, 'owned' => owned?(entry))
+          end
+        end
+      end
+
+      # Persist the approved source scope and bounded set of replacement pairs.
+      def start_migration(from_ref:, moves:)
+        finish_migration
+        current = migration
+        if current && current['from_ref'] != from_ref
+          raise Woods::ConfigurationError, 'Finish the pending Unblocked migration before selecting another source ref'
+        end
+
+        return if current.nil? && moves.empty?
+
+        @scopes.fetch(@active_scope)['migration'] ||= { 'from_ref' => from_ref, 'moves' => moves }
+      end
+
+      def migration
+        @scopes.dig(@active_scope, 'migration')
+      end
+
+      def complete_move(move)
+        source = if move.fetch('source_scope') == 'legacy'
+                   @legacy_documents
+                 else
+                   @scopes.dig(move.fetch('source_scope'), 'documents')
+                 end
+        source&.delete(move.fetch('old_uri'))
+        migration.fetch('moves').delete(move)
+      end
+
+      # Finish only after every selected replacement and old-copy cleanup.
+      def finish_migration
+        @scopes.fetch(@active_scope).delete('migration') if migration && migration.fetch('moves').empty?
+      end
+
       private
 
-      # Load the persisted documents, discarding data from a different
-      # collection, a different schema version, or an unparseable/unreadable
-      # file. Every discard warns to stderr — the consequence (a full re-push)
-      # is expensive enough that operators need to know why it happened.
-      #
-      # AtomicFile.read, not File.read: a bare read tags the bytes with the
-      # process's default external encoding (US-ASCII under LANG=C), so a
-      # non-ASCII document URI raised EncodingError out of JSON.parse instead
-      # of degrading. Read failures (e.g. Errno::EACCES) take the same discard
-      # path — the manifest is a cache, and "everything is new" is always a
-      # correct answer.
-      #
+      def scope_key(repo, ref)
+        Digest::SHA256.hexdigest(JSON.generate([repo, ref]))
+      end
+
+      def encode_ref(ref)
+        ref.split('/').map { |segment| ERB::Util.url_encode(segment) }.join('/')
+      end
+
+      def owned?(entry)
+        entry['owned'] == true || (entry['hash'].is_a?(String) && !entry['hash'].empty?)
+      end
+
+      def valid_documents?(documents)
+        documents.is_a?(Hash) && documents.all? { |uri, entry| uri.is_a?(String) && entry.is_a?(Hash) }
+      end
+
+      def valid_scopes?(scopes)
+        scopes.is_a?(Hash) && scopes.all? do |key, scope|
+          key.is_a?(String) && scope.is_a?(Hash) && scope['repo_url'].is_a?(String) &&
+            scope['ref'].is_a?(String) && key == scope_key(scope['repo_url'], scope['ref']) &&
+            valid_documents?(scope['documents']) && valid_migration?(scope['migration'])
+        end
+      end
+
+      def valid_migration?(migration)
+        return true if migration.nil?
+        unless migration.is_a?(Hash) && migration['from_ref'].is_a?(String) && migration['moves'].is_a?(Array)
+          return false
+        end
+
+        migration['moves'].all? do |move|
+          move.is_a?(Hash) && move['old_uri'].is_a?(String) && move['source_scope'].is_a?(String) &&
+            (move['new_uri'].nil? || move['new_uri'].is_a?(String)) && [true, false].include?(move['owned'])
+        end
+      end
+
+      # Retain legacy records, but refuse malformed ownership state at sync
+      # preflight. AtomicFile supplies UTF-8 independently of the host locale.
       # @return [Hash{String=>Hash}] uri => { 'hash' =>, 'document_id' => }
       def load
         return {} unless File.exist?(@path)
 
         parsed = JSON.parse(AtomicFile.read(@path))
         return discard('not a JSON object') unless parsed.is_a?(Hash)
-        return discard("schema version #{parsed['version'].inspect}, expected #{VERSION}") unless
-          parsed['version'] == VERSION
+        return discard("schema version #{parsed['version'].inspect}, expected 1 or #{VERSION}") unless
+          [1, VERSION].include?(parsed['version'])
         return discard("written for collection #{parsed['collection_id'].inspect}, expected #{@collection_id}") unless
           parsed['collection_id'] == @collection_id
 
+        @scopes = parsed.fetch('scopes', {})
+        return discard('invalid scoped ownership records') unless valid_scopes?(@scopes)
+
+        @active_scope = parsed['active_scope']
         documents = parsed['documents']
-        documents.is_a?(Hash) ? documents : {}
+        return discard('invalid document ownership records') unless valid_documents?(documents)
+
+        documents
       rescue JSON::ParserError
         discard('unparseable JSON')
       rescue EncodingError, SystemCallError => e
@@ -135,7 +248,10 @@ module Woods
       # @param reason [String] Why the persisted manifest is unusable
       # @return [Hash] empty documents hash (degrades to a full re-push)
       def discard(reason)
-        warn "WARNING: discarding sync manifest at #{@path} (#{reason}) — next sync re-pushes all documents"
+        @load_error = reason
+        @scopes = {}
+        warn "WARNING: discarding sync manifest at #{@path} (#{reason}) — " \
+             'sync requires manifest recovery before remote changes'
         {}
       end
     end