@effect-agent/storage-sqlite 0.1.0-beta.8 → 0.1.0-beta.80

package/src/index.ts

@@ -1,6 +1,10 @@
-export * from "./errors.ts";
-export * from "./migrations.ts";
-export * from "./sqlite-conversation-store.ts";
-export * from "./sqlite-ledger.ts";
-export * from "./sqlite-storage-config.ts";
-export * from "./sqlite-storage-failpoint.ts";
+export * as SqliteMessageDeliveryStore from "./SqliteMessageDeliveryStore.ts";
+export * as SqliteActivityStore from "./SqliteActivityStore.ts";
+export * as SqliteScheduleStore from "./SqliteScheduleStore.ts";
+export * as SqliteStorageConfig from "./SqliteStorageConfig.ts";
+export * as SqliteStorageError from "./SqliteStorageError.ts";
+export * as SqliteStorageFailpoint from "./SqliteStorageFailpoint.ts";
+export * as SqliteStorageVersion from "./SqliteStorageVersion.ts";
+export * as SqliteSubmissionLedger from "./SqliteSubmissionLedger.ts";
+export * as SqliteSubscriptionStore from "./SqliteSubscriptionStore.ts";
+export * as SqliteThreadStore from "./SqliteThreadStore.ts";

package/src/internal/message-delivery-schema.ts

@@ -0,0 +1,24 @@
+import { Effect } from "effect";
+import * as SqlClient from "effect/unstable/sql/SqlClient";
+
+/** Additive storage owned by this adapter; creation participates in its version transaction. */
+export const createMessageDeliveryTables = Effect.gen(function* () {
+  const sql = yield* SqlClient.SqlClient;
+
+  yield* sql`
+    CREATE TABLE effect_agent_message_deliveries (
+      owner_thread_id TEXT NOT NULL,
+      message_id TEXT NOT NULL,
+      version INTEGER NOT NULL,
+      state TEXT NOT NULL,
+      deadline_at_millis INTEGER,
+      record_json TEXT NOT NULL,
+      PRIMARY KEY (owner_thread_id, message_id)
+    )
+  `.withoutTransform;
+  yield* sql`
+    CREATE INDEX effect_agent_message_deliveries_due
+    ON effect_agent_message_deliveries (deadline_at_millis, owner_thread_id, message_id)
+    WHERE deadline_at_millis IS NOT NULL
+  `.withoutTransform;
+});

package/src/{migrations.ts → internal/migrations.ts}

@@ -2,15 +2,27 @@ import { SqliteMigrator } from "@effect/sql-sqlite-node";
 import { Effect } from "effect";
 import * as SqlClient from "effect/unstable/sql/SqlClient";
 
-export const CurrentSqliteStorageVersion = 4;
+import { createMessageDeliveryTables } from "./message-delivery-schema.ts";
+import { createRecoveryCheckpointTable } from "./recovery-checkpoint-schema.ts";
 
+export const CurrentSqliteStorageVersion = 11;
+
+/** Index only outstanding obligations, ordered by the recovery scan's stable cursor. */
+export const createNonterminalIndex = Effect.gen(function* () {
+  const sql = yield* SqlClient.SqlClient;
+
+  yield* sql`CREATE INDEX effect_agent_submissions_nonterminal ON effect_agent_submissions (thread_id, queue_sequence) WHERE state <> 'settled'`
+    .withoutTransform;
+});
+
+/** Initialize empty storage with the complete current schema. */
 export const sqliteMigrations = SqliteMigrator.fromRecord({
-  "1_current_persistent_conversation_foundation": Effect.gen(function* () {
+  "1_current_thread_storage": Effect.gen(function* () {
     const sql = yield* SqlClient.SqlClient;
 
     yield* sql`
-      CREATE TABLE effect_agent_conversations (
-        conversation_id TEXT PRIMARY KEY NOT NULL,
+      CREATE TABLE effect_agent_threads (
+        thread_id TEXT PRIMARY KEY NOT NULL,
         created_at TEXT NOT NULL,
         tail_sequence INTEGER NOT NULL,
         tail_digest TEXT NOT NULL,
@@ -20,64 +32,59 @@ export const sqliteMigrations = SqliteMigrator.fromRecord({
 
     yield* sql`
       CREATE TABLE effect_agent_canonical_batches (
-        conversation_id TEXT NOT NULL,
+        thread_id TEXT NOT NULL,
         batch_id TEXT NOT NULL,
         first_sequence INTEGER NOT NULL,
         last_sequence INTEGER NOT NULL,
         batch_digest TEXT NOT NULL,
         tail_digest TEXT NOT NULL,
         batch_json TEXT NOT NULL,
-        PRIMARY KEY (conversation_id, batch_id),
-        FOREIGN KEY (conversation_id)
-          REFERENCES effect_agent_conversations(conversation_id)
+        PRIMARY KEY (thread_id, batch_id),
+        FOREIGN KEY (thread_id)
+          REFERENCES effect_agent_threads(thread_id)
           ON DELETE RESTRICT
       )
     `.withoutTransform;
 
     yield* sql`
       CREATE TABLE effect_agent_canonical_records (
-        conversation_id TEXT NOT NULL,
+        thread_id TEXT NOT NULL,
         sequence INTEGER NOT NULL,
         record_id TEXT NOT NULL,
         batch_id TEXT NOT NULL,
         record_json TEXT NOT NULL,
-        PRIMARY KEY (conversation_id, sequence),
-        UNIQUE (conversation_id, record_id),
-        FOREIGN KEY (conversation_id, batch_id)
-          REFERENCES effect_agent_canonical_batches(conversation_id, batch_id)
+        PRIMARY KEY (thread_id, sequence),
+        UNIQUE (thread_id, record_id),
+        FOREIGN KEY (thread_id, batch_id)
+          REFERENCES effect_agent_canonical_batches(thread_id, batch_id)
           ON DELETE RESTRICT
       )
     `.withoutTransform;
 
     yield* sql`
       CREATE INDEX effect_agent_canonical_records_batch
-        ON effect_agent_canonical_records (conversation_id, batch_id, sequence)
+        ON effect_agent_canonical_records (thread_id, batch_id, sequence)
     `.withoutTransform;
 
     yield* sql`
       CREATE TABLE effect_agent_checkpoints (
-        conversation_id TEXT NOT NULL,
+        thread_id TEXT NOT NULL,
         through_sequence INTEGER NOT NULL,
         tail_digest TEXT NOT NULL,
         checkpoint_json TEXT NOT NULL,
-        PRIMARY KEY (conversation_id, through_sequence),
-        FOREIGN KEY (conversation_id)
-          REFERENCES effect_agent_conversations(conversation_id)
+        PRIMARY KEY (thread_id, through_sequence),
+        FOREIGN KEY (thread_id)
+          REFERENCES effect_agent_threads(thread_id)
           ON DELETE RESTRICT
       )
     `.withoutTransform;
 
-    yield* sql`PRAGMA user_version = 1`.withoutTransform;
-  }),
-  "2_durable_submission_ledger": Effect.gen(function* () {
-    const sql = yield* SqlClient.SqlClient;
-
-    // Admission rows exist before Conversation materialization (durability §4), so
-    // conversation_id intentionally carries no foreign key into effect_agent_conversations.
+    // Admission rows exist before Thread materialization (durability §4), so
+    // thread_id intentionally carries no foreign key into effect_agent_threads.
     yield* sql`
       CREATE TABLE effect_agent_submissions (
         submission_id TEXT PRIMARY KEY NOT NULL,
-        conversation_id TEXT NOT NULL,
+        thread_id TEXT NOT NULL,
         queue_sequence INTEGER NOT NULL,
         principal TEXT NOT NULL,
         idempotency_key TEXT NOT NULL,
@@ -93,11 +100,27 @@ export const sqliteMigrations = SqliteMigrator.fromRecord({
         ready_at TEXT,
         input_applied_record_id TEXT,
         input_applied_sequence INTEGER,
-        UNIQUE (conversation_id, principal, idempotency_key),
-        UNIQUE (conversation_id, queue_sequence)
+        joined_host_submission_id TEXT,
+        suspended_reason_json TEXT,
+        suspended_at TEXT,
+        unknown_reason TEXT,
+        unknown_tool_call_ids_json TEXT,
+        parent_submission_id TEXT,
+        parent_tool_call_id TEXT,
+        admission_group TEXT,
+        admission_fence_json TEXT,
+        worker_admission_json TEXT,
+        message_admission_json TEXT,
+        UNIQUE (thread_id, principal, idempotency_key),
+        UNIQUE (thread_id, queue_sequence)
       )
     `.withoutTransform;
 
+    yield* sql`
+      CREATE INDEX effect_agent_submissions_group
+        ON effect_agent_submissions (thread_id, admission_group, state)
+    `.withoutTransform;
+
     yield* sql`
       CREATE TABLE effect_agent_submission_ownership (
         submission_id TEXT PRIMARY KEY NOT NULL,
@@ -116,7 +139,7 @@ export const sqliteMigrations = SqliteMigrator.fromRecord({
       CREATE TABLE effect_agent_attempts (
         attempt_id TEXT PRIMARY KEY NOT NULL,
         submission_id TEXT NOT NULL,
-        conversation_id TEXT NOT NULL,
+        thread_id TEXT NOT NULL,
         owner_producer_id TEXT NOT NULL,
         producer_epoch INTEGER NOT NULL,
         claimed_at TEXT NOT NULL,
@@ -155,42 +178,6 @@ export const sqliteMigrations = SqliteMigrator.fromRecord({
       )
     `.withoutTransform;
 
-    yield* sql`PRAGMA user_version = 2`.withoutTransform;
-  }),
-  "3_durable_tools_and_joined_input": Effect.gen(function* () {
-    const sql = yield* SqlClient.SqlClient;
-
-    // Joined queued input (plan §2.5, DUR-016): a joining/joined Submission records which host
-    // Run claimed it. The canonical input marker reuses input_applied_record_id/_sequence
-    // because the joined input IS the Submission's own canonical `input:{sid}` record.
-    yield* sql`
-      ALTER TABLE effect_agent_submissions
-        ADD COLUMN joined_host_submission_id TEXT
-    `.withoutTransform;
-
-    // Durable approval suspension (plan §2.6): the reason is the Schema-encoded
-    // SuspensionReason union so later suspension families stay additive.
-    yield* sql`
-      ALTER TABLE effect_agent_submissions
-        ADD COLUMN suspended_reason_json TEXT
-    `.withoutTransform;
-    yield* sql`
-      ALTER TABLE effect_agent_submissions
-        ADD COLUMN suspended_at TEXT
-    `.withoutTransform;
-
-    // Unknown Outcome marking (DUR-009/DUR-017): the marked open Tool Call identities are
-    // stored so `recordUnknownResolution` can reopen the lane only when every marked call
-    // has a durable resolution intent.
-    yield* sql`
-      ALTER TABLE effect_agent_submissions
-        ADD COLUMN unknown_reason TEXT
-    `.withoutTransform;
-    yield* sql`
-      ALTER TABLE effect_agent_submissions
-        ADD COLUMN unknown_tool_call_ids_json TEXT
-    `.withoutTransform;
-
     yield* sql`
       CREATE INDEX effect_agent_submissions_joined_host
         ON effect_agent_submissions (joined_host_submission_id)
@@ -226,21 +213,8 @@ export const sqliteMigrations = SqliteMigrator.fromRecord({
       )
     `.withoutTransform;
 
-    yield* sql`PRAGMA user_version = 3`.withoutTransform;
-  }),
-  "4_durable_subagents": Effect.gen(function* () {
-    const sql = yield* SqlClient.SqlClient;
-
     // Durable attached children (spec §12, SUB-004): a child Submission records its immutable
     // parent linkage at admission; the parent-side index serves the recovery attachment view.
-    yield* sql`
-      ALTER TABLE effect_agent_submissions
-        ADD COLUMN parent_submission_id TEXT
-    `.withoutTransform;
-    yield* sql`
-      ALTER TABLE effect_agent_submissions
-        ADD COLUMN parent_tool_call_id TEXT
-    `.withoutTransform;
     yield* sql`
       CREATE INDEX effect_agent_submissions_parent
         ON effect_agent_submissions (parent_submission_id)
@@ -270,6 +244,109 @@ export const sqliteMigrations = SqliteMigrator.fromRecord({
       )
     `.withoutTransform;
 
-    yield* sql`PRAGMA user_version = 4`.withoutTransform;
+    // record_json is authoritative. The remaining columns support owner keyset paging and
+    // deadline queries without decoding unrelated future schedules.
+    yield* sql`
+      CREATE TABLE effect_agent_schedules (
+        tenant_id TEXT NOT NULL,
+        owner_id TEXT NOT NULL,
+        schedule_id TEXT NOT NULL,
+        deadline_at_millis INTEGER,
+        record_json TEXT NOT NULL,
+        PRIMARY KEY (tenant_id, owner_id, schedule_id)
+      )
+    `.withoutTransform;
+
+    yield* sql`
+      CREATE INDEX effect_agent_schedules_deadline
+        ON effect_agent_schedules (deadline_at_millis, tenant_id, owner_id, schedule_id)
+        WHERE deadline_at_millis IS NOT NULL
+    `.withoutTransform;
+
+    yield* sql`
+      CREATE INDEX effect_agent_schedules_owner_deadline
+        ON effect_agent_schedules (tenant_id, owner_id, deadline_at_millis, schedule_id)
+        WHERE deadline_at_millis IS NOT NULL
+    `.withoutTransform;
+
+    yield* sql`
+      CREATE TABLE effect_agent_subscription_sequences (
+        tenant_id TEXT NOT NULL,
+        source_address TEXT NOT NULL,
+        sequence INTEGER NOT NULL,
+        event_scan_cursor TEXT NOT NULL,
+        delivery_scan_cursor TEXT NOT NULL,
+        recovery_scan_cursor INTEGER NOT NULL,
+        PRIMARY KEY (tenant_id, source_address)
+      )
+    `.withoutTransform;
+    yield* sql`
+      CREATE TABLE effect_agent_subscriptions (
+        tenant_id TEXT NOT NULL,
+        source_address TEXT NOT NULL,
+        owner_id TEXT NOT NULL,
+        subscription_id TEXT NOT NULL,
+        ordinal INTEGER NOT NULL,
+        source_name TEXT NOT NULL,
+        source_version TEXT NOT NULL,
+        matching_key TEXT NOT NULL,
+        state TEXT NOT NULL,
+        expires_at_millis INTEGER,
+        recovery_at_millis INTEGER,
+        recovery_present INTEGER NOT NULL DEFAULT 0,
+        record_json TEXT NOT NULL,
+        PRIMARY KEY (tenant_id, source_address, owner_id, subscription_id),
+        UNIQUE (tenant_id, source_address, ordinal)
+      )
+    `.withoutTransform;
+    yield* sql`CREATE INDEX effect_agent_subscriptions_owner ON effect_agent_subscriptions (tenant_id, source_address, owner_id, ordinal)`
+      .withoutTransform;
+    yield* sql`CREATE INDEX effect_agent_subscriptions_candidates ON effect_agent_subscriptions (tenant_id, source_address, source_name, source_version, matching_key, ordinal)`
+      .withoutTransform;
+    yield* sql`CREATE INDEX effect_agent_subscriptions_recovery ON effect_agent_subscriptions (tenant_id, source_address, recovery_at_millis, ordinal) WHERE recovery_at_millis IS NOT NULL`
+      .withoutTransform;
+    yield* sql`
+      CREATE TABLE effect_agent_subscription_events (
+        tenant_id TEXT NOT NULL,
+        source_address TEXT NOT NULL,
+        event_id TEXT NOT NULL,
+        source_name TEXT NOT NULL,
+        source_version TEXT NOT NULL,
+        matching_key TEXT NOT NULL,
+        payload_digest TEXT NOT NULL,
+        cutoff INTEGER NOT NULL,
+        cursor INTEGER NOT NULL,
+        routing_complete INTEGER NOT NULL,
+        tombstone INTEGER NOT NULL DEFAULT 0,
+        next_attempt_at_millis INTEGER NOT NULL,
+        record_json TEXT NOT NULL,
+        PRIMARY KEY (tenant_id, source_address, event_id)
+      )
+    `.withoutTransform;
+    yield* sql`CREATE INDEX effect_agent_subscription_events_pending ON effect_agent_subscription_events (tenant_id, source_address, routing_complete, next_attempt_at_millis, event_id)`
+      .withoutTransform;
+    yield* sql`
+      CREATE TABLE effect_agent_subscription_deliveries (
+        tenant_id TEXT NOT NULL,
+        source_address TEXT NOT NULL,
+        owner_id TEXT NOT NULL,
+        subscription_id TEXT NOT NULL,
+        event_id TEXT NOT NULL,
+        delivery_key TEXT NOT NULL,
+        state TEXT NOT NULL,
+        next_attempt_at_millis INTEGER NOT NULL,
+        record_json TEXT NOT NULL,
+        PRIMARY KEY (tenant_id, source_address, owner_id, subscription_id, event_id),
+        UNIQUE (tenant_id, source_address, delivery_key)
+      )
+    `.withoutTransform;
+    yield* sql`CREATE INDEX effect_agent_subscription_deliveries_pending ON effect_agent_subscription_deliveries (tenant_id, source_address, state, next_attempt_at_millis, delivery_key)`
+      .withoutTransform;
+    yield* sql`CREATE INDEX effect_agent_subscription_deliveries_registration ON effect_agent_subscription_deliveries (tenant_id, source_address, owner_id, subscription_id, delivery_key)`
+      .withoutTransform;
+    yield* createNonterminalIndex;
+    yield* createMessageDeliveryTables;
+    yield* createRecoveryCheckpointTable;
+    yield* sql`PRAGMA user_version = 11`.withoutTransform;
   }),
 });

package/src/internal/recovery-checkpoint-schema.ts

@@ -0,0 +1,17 @@
+import { Effect } from "effect";
+import * as SqlClient from "effect/unstable/sql/SqlClient";
+
+/** One disposable recovery snapshot per Thread, independent of generic projections. */
+export const createRecoveryCheckpointTable = Effect.gen(function* () {
+  const sql = yield* SqlClient.SqlClient;
+
+  yield* sql`
+    CREATE TABLE effect_agent_recovery_checkpoints (
+      thread_id TEXT PRIMARY KEY NOT NULL,
+      through_sequence INTEGER NOT NULL,
+      tail_digest TEXT NOT NULL,
+      checkpoint_json TEXT NOT NULL,
+      FOREIGN KEY (thread_id) REFERENCES effect_agent_threads(thread_id) ON DELETE RESTRICT
+    )
+  `.withoutTransform;
+});