textus 0.55.2 → 0.56.0

data/lib/textus/infra/store/cursor_store.rb

@@ -0,0 +1,27 @@
+module Textus
+  module Infra
+    class Store
+      class CursorStore < Base
+        def get(role)
+          result(table(:cursors).where(role:).first) || { role:, seq: 0 }
+        end
+
+        def set(role, seq)
+          table(:cursors).insert_conflict(
+            update: { seq:, updated_at: Time.now.utc },
+          ).insert(role:, seq:, updated_at: Time.now.utc)
+        end
+
+        def get_loop(role)
+          result(table(:loop_positions).where(role:).first) || { role:, index: 0 }
+        end
+
+        def set_loop(role, index)
+          table(:loop_positions).insert_conflict(
+            update: { index:, updated_at: Time.now.utc },
+          ).insert(role:, index:, updated_at: Time.now.utc)
+        end
+      end
+    end
+  end
+end

data/lib/textus/infra/store/entry_index.rb

@@ -0,0 +1,31 @@
+module Textus
+  module Infra
+    class Store
+      class EntryIndex < Base
+        def record(key:, lane:, format:, schema:, content:)
+          table(:entries).insert_conflict(:replace).insert(
+            key:, lane:, format:, schema:, content:,
+            indexed_at: Time.now.utc
+          )
+        end
+
+        def remove(key:)
+          table(:entries).where(key:).delete
+        end
+
+        def search(query: nil, schema: nil, lane: nil, prefix: nil)
+          dataset = table(:entries).select(:key, :lane, :format, :schema, :content, :indexed_at)
+          if query
+            match_query = query.to_s.gsub("-", " ")
+            matching_rowids = @db[:entries_fts].where(Sequel.lit("entries_fts MATCH ?", match_query)).select_map(:rowid)
+            dataset = dataset.where(Sequel[:entries][:rowid] => matching_rowids)
+          end
+          dataset = dataset.where(schema:) if schema
+          dataset = dataset.where(lane:) if lane
+          dataset = dataset.where(Sequel.like(:key, "#{prefix}%")) if prefix
+          results(dataset.all)
+        end
+      end
+    end
+  end
+end

data/lib/textus/infra/store/envelope/meta.rb

@@ -0,0 +1,63 @@
+require "securerandom"
+
+module Textus
+  module Infra
+    class Store
+      module Envelope
+        module Meta
+          NO_META_FORMATS = %w[text].freeze
+
+          FIELDS = {
+            "uid" => {
+              inject: lambda { |meta, content, existing_meta, **_opts|
+                m = meta.is_a?(Hash) ? meta.dup : {}
+                existing = existing_meta.is_a?(Hash) ? existing_meta["uid"] : nil
+                m["uid"] = existing || Textus::Value::Uid.mint unless m["uid"].is_a?(String) && !m["uid"].empty?
+                [m, content]
+              },
+            },
+            "sources" => {
+              inject: lambda { |meta, content, existing_meta, etag_for: nil|
+                m = meta.is_a?(Hash) ? meta.dup : {}
+                existing = existing_meta.is_a?(Hash) ? existing_meta["sources"] : nil
+
+                if m.key?("sources")
+                  raise Textus::BadContent.new(nil, "_meta.sources must be an array") unless m["sources"].is_a?(Array)
+
+                  m["sources"] = m["sources"].map { |s| Meta.normalize_source!(s, etag_for) }
+                elsif existing.is_a?(Array) && !existing.empty?
+                  m["sources"] = existing
+                end
+
+                [m, content]
+              },
+            },
+          }.freeze
+
+          def self.inject_all(meta, content, existing_meta = {}, format: nil, etag_for: nil)
+            return [meta, content] if NO_META_FORMATS.include?(format)
+
+            FIELDS.each_value do |field|
+              meta, content = field[:inject].call(meta, content, existing_meta, etag_for: etag_for)
+            end
+
+            [meta, content]
+          end
+
+          def self.normalize_source!(src, etag_for)
+            key = case src
+                  when String then src
+                  when Hash   then src["key"]
+                  end
+
+            raise Textus::BadContent.new(nil, "each source must be a string key or { key: } object") unless key.is_a?(String)
+            raise Textus::BadContent.new(nil, "each source key must be a non-empty string") if key.empty?
+
+            etag = etag_for&.call(key)
+            etag ? { "key" => key, "etag" => etag } : { "key" => key }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/infra/store/file_store.rb

@@ -0,0 +1,23 @@
+module Textus
+  module Infra
+    class Store
+      class FileStore
+        def initialize(file_system)
+          @fs = file_system
+        end
+
+        def read(path)  = @fs.read(path)
+
+        def write(path, bytes)
+          @fs.mkdir_p(File.dirname(path))
+          @fs.write(path, bytes)
+        end
+
+        def delete(path)   = @fs.delete(path)
+        def move(from, to) = @fs.mv(from, to)
+        def exists?(path)  = @fs.exists?(path)
+        def etag(path)     = Textus::Value::Etag.for_file(path)
+      end
+    end
+  end
+end

data/lib/textus/infra/store/index/builder.rb

@@ -0,0 +1,28 @@
+module Textus
+  module Infra
+    class Store
+      module Index
+        class Builder
+          def initialize(db)
+            @db = db
+          end
+
+          def rebuild!(resolver:)
+            @db[:entries].each do |row|
+              key = row[:key]
+              mentry = resolver.resolve(key)
+              next unless mentry
+
+              entry = mentry.entry
+              @db[:entries].insert_conflict(:replace).insert(
+                key:, lane: entry.lane, format: entry.format,
+                schema: entry.schema, content: row[:content],
+                indexed_at: Time.now.utc
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/infra/store/index/lookup.rb

@@ -0,0 +1,23 @@
+module Textus
+  module Infra
+    class Store
+      module Index
+        class Lookup
+          def initialize(db)
+            @db = db
+          end
+
+          def find_by_hash(hash)
+            row = @db[:entries].where(Sequel.lit("json_extract(content, '$.content_hash') = ?", hash)).first
+            row&.dig(:key)
+          end
+
+          def find_by_url(url)
+            row = @db[:entries].where(Sequel.lit("json_extract(content, '$.source.url') = ?", url)).first
+            row&.dig(:key)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/infra/store/link_store.rb

@@ -0,0 +1,43 @@
+module Textus
+  module Infra
+    class Store
+      class LinkStore
+        def initialize(database)
+          @db = database
+        end
+
+        def add_edge(from_key:, to_key:)
+          @db[:link_edges].insert_ignore.insert(from_key:, to_key:)
+        end
+        alias record add_edge
+
+        def remove_edge(from_key:, to_key:)
+          @db[:link_edges].where(from_key:, to_key:).delete
+        end
+
+        def links_from(key)
+          @db[:link_edges].where(from_key: key).select_map(:to_key)
+        end
+
+        def links_to(key)
+          @db[:link_edges].where(to_key: key).select_map(:from_key)
+        end
+
+        def reachable(from:, depth: nil)
+          visited = Set.new
+          queue = [[from, 0]]
+          while queue.any?
+            current, d = queue.shift
+            next if visited.include?(current)
+            next if depth && d > depth
+
+            visited.add(current)
+            links_from(current).each { |to| queue << [to, d + 1] }
+          end
+          visited.delete(from)
+          visited.to_a
+        end
+      end
+    end
+  end
+end

data/lib/textus/infra/store/workflow_queue.rb

@@ -0,0 +1,54 @@
+module Textus
+  module Infra
+    class Store
+      class WorkflowQueue < Base
+        def push(event_type:, workflow_type:, key:, lane:, payload: nil,
+                 created_at: Time.now.utc, scheduled_at: nil,
+                 state: "pending", retry_count: 0, max_attempts: 3)
+          table(:workflow_events).insert(
+            event_type: event_type.to_s, workflow_type:, key:, lane:,
+            payload: payload&.to_json, created_at:, scheduled_at:,
+            state:, retry_count:, max_attempts:
+          )
+        end
+
+        def pop
+          row = table(:workflow_events)
+                .where(state: "pending")
+                .where(Sequel.|({ scheduled_at: nil }, Sequel.lit("scheduled_at <= datetime('now')")))
+                .order(:created_at, :id)
+                .limit(1)
+                .for_update
+                .first
+          result(row)
+        end
+
+        def pending?(event_type:, key:)
+          table(:workflow_events)
+            .where(event_type: event_type.to_s, key: key)
+            .where(state: %w[pending processing])
+            .any?
+        end
+
+        def mark_processing(event_id)
+          table(:workflow_events).where(id: event_id).update(state: "processing")
+        end
+
+        def complete(event_id)
+          table(:workflow_events).where(id: event_id).update(state: "done")
+        end
+
+        def fail(event_id, retry_count:, error:)
+          table(:workflow_events).where(id: event_id).update(state: "failed", retry_count:, error:)
+        end
+
+        def reschedule(event_id, retry_count:, delay:, error:)
+          table(:workflow_events).where(id: event_id).update(
+            state: "pending", retry_count:, error:,
+            scheduled_at: Sequel.lit("datetime('now', '+#{delay.to_i} seconds')")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/textus/infra/store.rb

@@ -0,0 +1,29 @@
+module Textus
+  module Infra
+    class Store
+      def initialize(database:, file_system:)
+        @db = database.connection
+        @file_system = file_system
+      end
+
+      attr_reader :file_system, :db
+
+      def cursor_store   = @cursor_store ||= CursorStore.new(@db)
+      def link_store     = @link_store ||= LinkStore.new(@db)
+      def entry_index    = @entry_index ||= EntryIndex.new(@db)
+      def audit_log      = @audit_log ||= AuditLog.new(@db)
+      def workflow_queue = @workflow_queue ||= WorkflowQueue.new(@db)
+      def file_store     = @file_store     ||= FileStore.new(@file_system)
+
+      def transaction(&) = @db.transaction(&)
+    end
+  end
+end
+
+require_relative "store/base"
+require_relative "store/cursor_store"
+require_relative "store/audit_log"
+require_relative "store/workflow_queue"
+require_relative "store/entry_index"
+require_relative "store/file_store"
+require_relative "store/link_store"

data/lib/textus/ingest.rb

@@ -0,0 +1,8 @@
+module Textus
+  module Ingest
+  end
+end
+
+require_relative "lanes/ingest/resolver"
+require_relative "lanes/ingest/entry_types"
+require_relative "lanes/ingest/handlers"

data/lib/textus/lanes/artifact/cache.rb

@@ -0,0 +1,35 @@
+module Textus
+  module Lanes
+    module Artifact
+      class Cache
+        def initialize(store, ttl_default: 3600 * 24 * 30)
+          @store = store
+          @ttl_default = ttl_default
+        end
+
+        def ttl(key)
+          entry = @store.get(key:)
+          return @ttl_default unless entry
+
+          entry["ttl"] || @ttl_default
+        end
+
+        def stale?(key)
+          entry = @store.get(key:)
+          return true unless entry
+          return true if entry["stale"] || entry["fetching"]
+
+          generated_at = entry["generated_at"]
+          return true unless generated_at
+
+          age = Time.now - Time.parse(generated_at)
+          age > ttl(key)
+        end
+
+        def expire(key)
+          @store.put(key:, meta: { "stale" => true })
+        end
+      end
+    end
+  end
+end

data/lib/textus/lanes/artifact/handlers.rb

@@ -0,0 +1,29 @@
+module Textus
+  module Lanes
+    module Artifact
+      module Handlers
+        module_function
+
+        def get(key:, store:, ctx:, call:)
+          store.get(key:)
+        end
+
+        def list(store:, ctx:, call:, prefix: nil)
+          store.list(prefix: prefix || "artifact")
+        end
+
+        def status(store:, ctx:, call:)
+          store.list(prefix: "artifact").map do |row|
+            cache = Cache.new(store)
+            { key: row[:key], stale: cache.stale?(row[:key]) }
+          end
+        end
+
+        def refresh(key:, store:, ctx:, call:)
+          cache = Cache.new(store)
+          cache.expire(key)
+        end
+      end
+    end
+  end
+end

data/lib/textus/lanes/artifact/lifecycle.rb

@@ -0,0 +1,23 @@
+module Textus
+  module Lanes
+    module Artifact
+      class Lifecycle
+        def initialize(cache)
+          @cache = cache
+        end
+
+        def on_stale(key)
+          @cache.expire(key)
+        end
+
+        def on_expire(key)
+          @cache.expire(key)
+        end
+
+        def retention(key)
+          @cache.ttl(key)
+        end
+      end
+    end
+  end
+end

data/lib/textus/lanes/artifact/verbs.rb

@@ -0,0 +1,18 @@
+module Textus
+  module Lanes
+    module Artifact
+      module Verbs
+        Textus::Protocol::VerbRegistry.register(
+          Textus::Protocol::VerbRegistry::VerbSpec.new(
+            :workflow_spec, "Return the full definition of a named workflow — match, steps, publish targets.",
+            [Textus::Protocol::VerbRegistry::ArgSpec.arg(
+              name: :name, required: true, positional: true,
+              description: "workflow type name (e.g. 'boot', 'decisions-log', 'materialize')"
+            )],
+            %i[cli mcp], :read
+          ),
+        )
+      end
+    end
+  end
+end

data/lib/textus/lanes/ingest/dedup.rb

@@ -0,0 +1,17 @@
+module Textus
+  module Lanes
+    module Ingest
+      module Dedup
+        module_function
+
+        def find(lookup, content_hash, url: nil)
+          dup = lookup.find_by_hash(content_hash)
+          return dup if dup
+          return nil unless url
+
+          lookup.find_by_url(url)
+        end
+      end
+    end
+  end
+end

data/lib/textus/lanes/ingest/entry_types/asset.rb

@@ -0,0 +1,45 @@
+module Textus
+  module Lanes
+    module Ingest
+      module EntryTypes
+        module Asset
+          module_function
+
+          def build(path:, lane:, time:, layout:, label: nil, **)
+            asset_rel = copy(path, lane:, time:, layout:)
+            { "source" => { "kind" => "asset", "label" => label || File.basename(path) },
+              "asset" => asset_rel, "ingested_at" => time.utc.iso8601, "body" => nil }
+          end
+
+          def copy(path, lane:, time:, layout:)
+            date_path = time.strftime("%Y/%m/%d")
+            filename  = File.basename(path)
+            assets_dir = layout.asset_raw_dir(date_path, lane)
+            FileUtils.mkdir_p(assets_dir)
+            FileUtils.cp(path, File.join(assets_dir, filename))
+            sentinel = layout.asset_sentinel_path
+            File.write(sentinel, "*\n") unless File.exist?(sentinel)
+            "raw/#{date_path}/#{lane}/#{filename}"
+          end
+
+          def move(old_rel, lane, layout)
+            old_path = layout.asset_resolve(old_rel)
+            return unless File.exist?(old_path)
+
+            now = Time.now.utc
+            date_path = now.strftime("%Y/%m/%d")
+            filename = File.basename(old_path)
+            new_dir = layout.asset_raw_dir(date_path, lane)
+            new_path = File.join(new_dir, filename)
+            return if old_path == new_path
+
+            FileUtils.mkdir_p(new_dir)
+            FileUtils.mv(old_path, new_path)
+          rescue Errno::ENOENT, Errno::EACCES => e
+            warn "[textus ingest] could not move asset #{old_rel}: #{e.message}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/lanes/ingest/entry_types/link.rb

@@ -0,0 +1,16 @@
+module Textus
+  module Lanes
+    module Ingest
+      module EntryTypes
+        module Link
+          module_function
+
+          def build(url:, time:, label: nil, **)
+            { "source" => { "kind" => "url", "url" => url, "label" => label || url },
+              "ingested_at" => time.utc.iso8601, "body" => nil }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/lanes/ingest/entry_types/text.rb

@@ -0,0 +1,17 @@
+module Textus
+  module Lanes
+    module Ingest
+      module EntryTypes
+        module Text
+          module_function
+
+          def build(path:, time:, label: nil, **)
+            { "source" => { "kind" => "file", "path" => path,
+                            "label" => label || File.basename(path) },
+              "ingested_at" => time.utc.iso8601, "body" => File.read(path) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/lanes/ingest/entry_types.rb

@@ -0,0 +1,25 @@
+module Textus
+  module Lanes
+    module Ingest
+      module EntryTypes
+        require_relative "entry_types/link"
+        require_relative "entry_types/asset"
+        require_relative "entry_types/text"
+
+        module_function
+
+        def for(kind)
+          registry[kind] or raise(ArgumentError.new("unknown ingest kind: #{kind}"))
+        end
+
+        def registry
+          @registry ||= {
+            "link" => Link,
+            "asset" => Asset,
+            "text" => Text,
+          }.freeze
+        end
+      end
+    end
+  end
+end

data/lib/textus/lanes/ingest/handlers.rb

@@ -0,0 +1,54 @@
+module Textus
+  module Lanes
+    module Ingest
+      module Handlers
+        SOURCE_KINDS = %w[url file asset].freeze
+
+        module_function
+
+        def ingest(kind:, slug:, ctx:, call:, url: nil, path: nil, lane: nil, label: nil)
+          raise Textus::UsageError.new("kind must be one of #{SOURCE_KINDS.join("|")}") unless SOURCE_KINDS.include?(kind)
+
+          case kind
+          when "url"   then raise Textus::UsageError.new("url required") unless url
+          when "file"  then raise Textus::UsageError.new("path required") unless path
+          when "asset"
+            raise Textus::UsageError.new("path required") unless path
+            raise Textus::UsageError.new("lane required") unless lane
+          end
+
+          now = Time.now.utc
+          key = KeyBuilder.derive_key(now, kind, slug)
+          content_hash = KeyBuilder.compute_hash(kind:, url:, path:)
+
+          entry_type = EntryTypes.for(kind)
+          time = now
+          structured = entry_type.build(url:, path:, lane:, label:, time:, layout: ctx.layout)
+
+          lookup = Textus::Infra::Store::Index::Lookup.new(ctx.container.store.db)
+          duplicate_key = Dedup.find(lookup, content_hash, url:)
+
+          if duplicate_key && duplicate_key != key
+            old_content = ctx.read(key: duplicate_key)&.content || {}
+            Resolver.supersede(duplicate_key, key, ctx:, call:)
+
+            EntryTypes::Asset.move(old_content["asset"], lane, ctx.layout) if (kind == "asset") && old_content["asset"]
+
+            old_count = old_content["ingest_count"] || 1
+            structured["ingest_count"] = old_count + 1
+            structured["duplicated_at"] = now.utc.iso8601
+            structured["supersedes"] = duplicate_key
+          else
+            structured["ingest_count"] = 1
+          end
+
+          env = ctx.put(key:, meta: {}, body: nil, content: structured, call:)
+
+          IndexRebuilder.rebuild!(ctx.container.store.db, ctx.manifest.resolver)
+
+          { "key" => key, "uid" => env.uid, "etag" => env.etag }
+        end
+      end
+    end
+  end
+end

data/lib/textus/lanes/ingest/index_rebuilder.rb

@@ -0,0 +1,13 @@
+module Textus
+  module Lanes
+    module Ingest
+      module IndexRebuilder
+        module_function
+
+        def rebuild!(db, resolver)
+          Textus::Infra::Store::Index::Builder.new(db).rebuild!(resolver:)
+        end
+      end
+    end
+  end
+end

data/lib/textus/lanes/ingest/key_builder.rb

@@ -0,0 +1,33 @@
+module Textus
+  module Lanes
+    module Ingest
+      module KeyBuilder
+        CONTENT_HASH_ALGO = "sha256"
+
+        module_function
+
+        def derive_key(now, kind, slug)
+          date = now.strftime("%Y.%m.%d")
+          "raw.#{date}.#{kind}-#{slug}"
+        end
+
+        def compute_hash(kind:, url: nil, path: nil)
+          digest = Digest::SHA256.new
+          case kind
+          when "url"
+            raise ArgumentError.new("url required") unless url
+
+            digest.update(url)
+          when "file", "asset"
+            raise ArgumentError.new("path required") unless path
+
+            digest.file(path)
+          else
+            raise ArgumentError.new("unknown kind: #{kind}")
+          end
+          "#{CONTENT_HASH_ALGO}:#{digest.hexdigest}"
+        end
+      end
+    end
+  end
+end