@retinue/agentkit 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/README.md +310 -0
- package/dist/adapters/bullmq/consumer.d.ts +33 -0
- package/dist/adapters/bullmq/consumer.js +41 -0
- package/dist/adapters/bullmq/dispatcher.d.ts +74 -0
- package/dist/adapters/bullmq/dispatcher.js +160 -0
- package/dist/adapters/bullmq/export.d.ts +31 -0
- package/dist/adapters/bullmq/export.js +53 -0
- package/dist/adapters/bullmq/extraction.d.ts +42 -0
- package/dist/adapters/bullmq/extraction.js +63 -0
- package/dist/adapters/bullmq/index.d.ts +13 -0
- package/dist/adapters/bullmq/index.js +13 -0
- package/dist/adapters/bullmq/lock.d.ts +77 -0
- package/dist/adapters/bullmq/lock.js +126 -0
- package/dist/adapters/bullmq/queue.d.ts +50 -0
- package/dist/adapters/bullmq/queue.js +81 -0
- package/dist/adapters/memory/artifact-exports.d.ts +11 -0
- package/dist/adapters/memory/artifact-exports.js +102 -0
- package/dist/adapters/memory/artifacts.d.ts +15 -0
- package/dist/adapters/memory/artifacts.js +134 -0
- package/dist/adapters/memory/blobs.d.ts +7 -0
- package/dist/adapters/memory/blobs.js +27 -0
- package/dist/adapters/memory/evaluation.d.ts +18 -0
- package/dist/adapters/memory/evaluation.js +148 -0
- package/dist/adapters/memory/files.d.ts +27 -0
- package/dist/adapters/memory/files.js +0 -0
- package/dist/adapters/memory/flows.d.ts +16 -0
- package/dist/adapters/memory/flows.js +117 -0
- package/dist/adapters/memory/hitl.d.ts +9 -0
- package/dist/adapters/memory/hitl.js +130 -0
- package/dist/adapters/memory/idempotency.d.ts +13 -0
- package/dist/adapters/memory/idempotency.js +32 -0
- package/dist/adapters/memory/index.d.ts +39 -0
- package/dist/adapters/memory/index.js +107 -0
- package/dist/adapters/memory/knowledge.d.ts +43 -0
- package/dist/adapters/memory/knowledge.js +248 -0
- package/dist/adapters/memory/mcp.d.ts +9 -0
- package/dist/adapters/memory/mcp.js +37 -0
- package/dist/adapters/memory/message-store.d.ts +17 -0
- package/dist/adapters/memory/message-store.js +70 -0
- package/dist/adapters/memory/principal-memory.d.ts +7 -0
- package/dist/adapters/memory/principal-memory.js +83 -0
- package/dist/adapters/memory/runtime.d.ts +29 -0
- package/dist/adapters/memory/runtime.js +0 -0
- package/dist/adapters/memory/sessions.d.ts +29 -0
- package/dist/adapters/memory/sessions.js +0 -0
- package/dist/adapters/memory/skills.d.ts +10 -0
- package/dist/adapters/memory/skills.js +41 -0
- package/dist/adapters/memory/thread-summary.d.ts +7 -0
- package/dist/adapters/memory/thread-summary.js +29 -0
- package/dist/adapters/memory/usage-limits.d.ts +13 -0
- package/dist/adapters/memory/usage-limits.js +72 -0
- package/dist/adapters/memory/usage.d.ts +16 -0
- package/dist/adapters/memory/usage.js +279 -0
- package/dist/adapters/otel/index.d.ts +111 -0
- package/dist/adapters/otel/index.js +133 -0
- package/dist/adapters/postgres/artifact-exports.d.ts +12 -0
- package/dist/adapters/postgres/artifact-exports.js +117 -0
- package/dist/adapters/postgres/artifacts.d.ts +16 -0
- package/dist/adapters/postgres/artifacts.js +172 -0
- package/dist/adapters/postgres/checkpoint-store.d.ts +16 -0
- package/dist/adapters/postgres/checkpoint-store.js +34 -0
- package/dist/adapters/postgres/config.d.ts +15 -0
- package/dist/adapters/postgres/config.js +187 -0
- package/dist/adapters/postgres/conversation-store.d.ts +4 -0
- package/dist/adapters/postgres/conversation-store.js +82 -0
- package/dist/adapters/postgres/evaluation.d.ts +17 -0
- package/dist/adapters/postgres/evaluation.js +193 -0
- package/dist/adapters/postgres/file-content.d.ts +30 -0
- package/dist/adapters/postgres/file-content.js +111 -0
- package/dist/adapters/postgres/files.d.ts +19 -0
- package/dist/adapters/postgres/files.js +209 -0
- package/dist/adapters/postgres/flows.d.ts +20 -0
- package/dist/adapters/postgres/flows.js +206 -0
- package/dist/adapters/postgres/hitl.d.ts +5 -0
- package/dist/adapters/postgres/hitl.js +247 -0
- package/dist/adapters/postgres/index.d.ts +35 -0
- package/dist/adapters/postgres/index.js +35 -0
- package/dist/adapters/postgres/knowledge.d.ts +48 -0
- package/dist/adapters/postgres/knowledge.js +255 -0
- package/dist/adapters/postgres/memory.d.ts +14 -0
- package/dist/adapters/postgres/memory.js +194 -0
- package/dist/adapters/postgres/message-store.d.ts +11 -0
- package/dist/adapters/postgres/message-store.js +145 -0
- package/dist/adapters/postgres/migrations.d.ts +69 -0
- package/dist/adapters/postgres/migrations.js +1594 -0
- package/dist/adapters/postgres/pg-executor.d.ts +19 -0
- package/dist/adapters/postgres/pg-executor.js +32 -0
- package/dist/adapters/postgres/retention.d.ts +26 -0
- package/dist/adapters/postgres/retention.js +59 -0
- package/dist/adapters/postgres/rollups.d.ts +17 -0
- package/dist/adapters/postgres/rollups.js +267 -0
- package/dist/adapters/postgres/run-coordinator.d.ts +5 -0
- package/dist/adapters/postgres/run-coordinator.js +98 -0
- package/dist/adapters/postgres/run-event-log.d.ts +26 -0
- package/dist/adapters/postgres/run-event-log.js +30 -0
- package/dist/adapters/postgres/run-store.d.ts +4 -0
- package/dist/adapters/postgres/run-store.js +199 -0
- package/dist/adapters/postgres/schema.d.ts +39 -0
- package/dist/adapters/postgres/schema.js +70 -0
- package/dist/adapters/postgres/session-state.d.ts +7 -0
- package/dist/adapters/postgres/session-state.js +99 -0
- package/dist/adapters/postgres/sql.d.ts +8 -0
- package/dist/adapters/postgres/sql.js +2 -0
- package/dist/adapters/postgres/transaction.d.ts +37 -0
- package/dist/adapters/postgres/transaction.js +93 -0
- package/dist/adapters/postgres/unit-of-work.d.ts +18 -0
- package/dist/adapters/postgres/unit-of-work.js +8 -0
- package/dist/adapters/postgres/usage-limits.d.ts +15 -0
- package/dist/adapters/postgres/usage-limits.js +136 -0
- package/dist/adapters/postgres/usage.d.ts +15 -0
- package/dist/adapters/postgres/usage.js +226 -0
- package/dist/adapters/redis/index.d.ts +9 -0
- package/dist/adapters/redis/index.js +9 -0
- package/dist/adapters/redis/realtime.d.ts +74 -0
- package/dist/adapters/redis/realtime.js +112 -0
- package/dist/adapters/supabase/index.d.ts +88 -0
- package/dist/adapters/supabase/index.js +84 -0
- package/dist/adapters/supabase/realtime.d.ts +30 -0
- package/dist/adapters/supabase/realtime.js +53 -0
- package/dist/adapters/supabase/rls.d.ts +99 -0
- package/dist/adapters/supabase/rls.js +216 -0
- package/dist/adapters/supabase/storage.d.ts +50 -0
- package/dist/adapters/supabase/storage.js +207 -0
- package/dist/agents/agent.d.ts +66 -0
- package/dist/agents/agent.js +209 -0
- package/dist/agents/define.d.ts +21 -0
- package/dist/agents/define.js +63 -0
- package/dist/agents/engine.d.ts +98 -0
- package/dist/agents/engine.js +462 -0
- package/dist/agents/index.d.ts +50 -0
- package/dist/agents/index.js +17 -0
- package/dist/artifacts/index.d.ts +114 -0
- package/dist/artifacts/index.js +219 -0
- package/dist/authorization/index.d.ts +76 -0
- package/dist/authorization/index.js +76 -0
- package/dist/capabilities/index.d.ts +120 -0
- package/dist/capabilities/index.js +167 -0
- package/dist/capabilities/runtime.d.ts +89 -0
- package/dist/capabilities/runtime.js +84 -0
- package/dist/citations/index.d.ts +161 -0
- package/dist/citations/index.js +182 -0
- package/dist/context/assembler.d.ts +82 -0
- package/dist/context/assembler.js +129 -0
- package/dist/context/compaction.d.ts +45 -0
- package/dist/context/compaction.js +55 -0
- package/dist/context/index.d.ts +75 -0
- package/dist/context/index.js +17 -0
- package/dist/core/content-parts.d.ts +194 -0
- package/dist/core/content-parts.js +23 -0
- package/dist/core/context.d.ts +51 -0
- package/dist/core/context.js +9 -0
- package/dist/core/env.d.ts +25 -0
- package/dist/core/env.js +41 -0
- package/dist/core/errors.d.ts +30 -0
- package/dist/core/errors.js +65 -0
- package/dist/core/events.d.ts +139 -0
- package/dist/core/events.js +99 -0
- package/dist/core/ids.d.ts +52 -0
- package/dist/core/ids.js +13 -0
- package/dist/core/index.d.ts +9 -0
- package/dist/core/index.js +9 -0
- package/dist/core/tokens.d.ts +22 -0
- package/dist/core/tokens.js +22 -0
- package/dist/core/validation.d.ts +34 -0
- package/dist/core/validation.js +176 -0
- package/dist/documents/extraction.d.ts +121 -0
- package/dist/documents/extraction.js +293 -0
- package/dist/documents/index.d.ts +199 -0
- package/dist/documents/index.js +65 -0
- package/dist/documents/parsers/pdf.d.ts +47 -0
- package/dist/documents/parsers/pdf.js +508 -0
- package/dist/documents/parsers/text.d.ts +59 -0
- package/dist/documents/parsers/text.js +325 -0
- package/dist/documents/read-tool.d.ts +52 -0
- package/dist/documents/read-tool.js +109 -0
- package/dist/documents/render.d.ts +29 -0
- package/dist/documents/render.js +59 -0
- package/dist/documents/vision.d.ts +159 -0
- package/dist/documents/vision.js +214 -0
- package/dist/entries/adapters-bullmq.d.ts +8 -0
- package/dist/entries/adapters-bullmq.js +8 -0
- package/dist/entries/adapters-otel.d.ts +13 -0
- package/dist/entries/adapters-otel.js +13 -0
- package/dist/entries/adapters-postgres.d.ts +10 -0
- package/dist/entries/adapters-postgres.js +10 -0
- package/dist/entries/adapters-redis.d.ts +3 -0
- package/dist/entries/adapters-redis.js +3 -0
- package/dist/entries/context.d.ts +20 -0
- package/dist/entries/context.js +20 -0
- package/dist/entries/flows.d.ts +15 -0
- package/dist/entries/flows.js +15 -0
- package/dist/entries/hitl.d.ts +10 -0
- package/dist/entries/hitl.js +10 -0
- package/dist/entries/knowledge.d.ts +18 -0
- package/dist/entries/knowledge.js +19 -0
- package/dist/entries/mcp.d.ts +10 -0
- package/dist/entries/mcp.js +10 -0
- package/dist/entries/observability.d.ts +14 -0
- package/dist/entries/observability.js +16 -0
- package/dist/entries/persistence.d.ts +11 -0
- package/dist/entries/persistence.js +11 -0
- package/dist/entries/providers.d.ts +14 -0
- package/dist/entries/providers.js +14 -0
- package/dist/entries/runtime.d.ts +13 -0
- package/dist/entries/runtime.js +13 -0
- package/dist/entries/server.d.ts +24 -0
- package/dist/entries/server.js +24 -0
- package/dist/entries/tools.d.ts +21 -0
- package/dist/entries/tools.js +21 -0
- package/dist/entries/usage.d.ts +10 -0
- package/dist/entries/usage.js +10 -0
- package/dist/evaluation/gate.d.ts +168 -0
- package/dist/evaluation/gate.js +180 -0
- package/dist/evaluation/graders.d.ts +125 -0
- package/dist/evaluation/graders.js +203 -0
- package/dist/evaluation/index.d.ts +120 -0
- package/dist/evaluation/index.js +183 -0
- package/dist/evaluation/judge.d.ts +75 -0
- package/dist/evaluation/judge.js +111 -0
- package/dist/export/index.d.ts +162 -0
- package/dist/export/index.js +363 -0
- package/dist/export/markdown.d.ts +19 -0
- package/dist/export/markdown.js +29 -0
- package/dist/export/pdf.d.ts +73 -0
- package/dist/export/pdf.js +407 -0
- package/dist/files/context.d.ts +97 -0
- package/dist/files/context.js +185 -0
- package/dist/files/index.d.ts +210 -0
- package/dist/files/index.js +338 -0
- package/dist/files/read-tool.d.ts +81 -0
- package/dist/files/read-tool.js +163 -0
- package/dist/files/turn-parts.d.ts +96 -0
- package/dist/files/turn-parts.js +171 -0
- package/dist/flows/index.d.ts +270 -0
- package/dist/flows/index.js +62 -0
- package/dist/flows/interpreter.d.ts +146 -0
- package/dist/flows/interpreter.js +426 -0
- package/dist/flows/runner.d.ts +145 -0
- package/dist/flows/runner.js +270 -0
- package/dist/graphql/index.d.ts +8 -0
- package/dist/graphql/index.js +8 -0
- package/dist/graphql/resolvers.d.ts +237 -0
- package/dist/graphql/resolvers.js +253 -0
- package/dist/graphql/schema.d.ts +11 -0
- package/dist/graphql/schema.js +258 -0
- package/dist/graphql/sse.d.ts +77 -0
- package/dist/graphql/sse.js +100 -0
- package/dist/hitl/approved-execution.d.ts +127 -0
- package/dist/hitl/approved-execution.js +177 -0
- package/dist/hitl/index.d.ts +79 -0
- package/dist/hitl/index.js +12 -0
- package/dist/hitl/service.d.ts +221 -0
- package/dist/hitl/service.js +268 -0
- package/dist/idempotency/index.d.ts +70 -0
- package/dist/idempotency/index.js +59 -0
- package/dist/index.d.ts +103 -0
- package/dist/index.js +60 -0
- package/dist/knowledge/chunking.d.ts +57 -0
- package/dist/knowledge/chunking.js +158 -0
- package/dist/knowledge/index.d.ts +119 -0
- package/dist/knowledge/index.js +166 -0
- package/dist/knowledge/retrieval.d.ts +146 -0
- package/dist/knowledge/retrieval.js +170 -0
- package/dist/loadtest/harness.d.ts +168 -0
- package/dist/loadtest/harness.js +507 -0
- package/dist/loadtest/index.d.ts +13 -0
- package/dist/loadtest/index.js +13 -0
- package/dist/loadtest/injection.d.ts +89 -0
- package/dist/loadtest/injection.js +147 -0
- package/dist/loadtest/metrics.d.ts +197 -0
- package/dist/loadtest/metrics.js +160 -0
- package/dist/loadtest/runbooks.d.ts +28 -0
- package/dist/loadtest/runbooks.js +159 -0
- package/dist/loadtest/scenario.d.ts +104 -0
- package/dist/loadtest/scenario.js +208 -0
- package/dist/mcp/egress.d.ts +53 -0
- package/dist/mcp/egress.js +115 -0
- package/dist/mcp/index.d.ts +93 -0
- package/dist/mcp/index.js +33 -0
- package/dist/mcp/provider.d.ts +62 -0
- package/dist/mcp/provider.js +0 -0
- package/dist/models/index.d.ts +98 -0
- package/dist/models/index.js +74 -0
- package/dist/models/pricing.d.ts +24 -0
- package/dist/models/pricing.js +37 -0
- package/dist/models/provider-factory.d.ts +31 -0
- package/dist/models/provider-factory.js +67 -0
- package/dist/models/streaming.d.ts +145 -0
- package/dist/models/streaming.js +272 -0
- package/dist/models/vision.d.ts +38 -0
- package/dist/models/vision.js +62 -0
- package/dist/persistence/index.d.ts +1654 -0
- package/dist/persistence/index.js +226 -0
- package/dist/principal-memory/index.d.ts +106 -0
- package/dist/principal-memory/index.js +89 -0
- package/dist/retention/index.d.ts +89 -0
- package/dist/retention/index.js +70 -0
- package/dist/runtime/checkpoint.d.ts +37 -0
- package/dist/runtime/checkpoint.js +22 -0
- package/dist/runtime/index.d.ts +118 -0
- package/dist/runtime/index.js +69 -0
- package/dist/runtime/retry.d.ts +95 -0
- package/dist/runtime/retry.js +126 -0
- package/dist/runtime/serialization.d.ts +85 -0
- package/dist/runtime/serialization.js +95 -0
- package/dist/runtime/streaming.d.ts +54 -0
- package/dist/runtime/streaming.js +115 -0
- package/dist/runtime/worker.d.ts +130 -0
- package/dist/runtime/worker.js +405 -0
- package/dist/security/checklist.d.ts +53 -0
- package/dist/security/checklist.js +204 -0
- package/dist/security/findings.d.ts +56 -0
- package/dist/security/findings.js +168 -0
- package/dist/security/index.d.ts +14 -0
- package/dist/security/index.js +14 -0
- package/dist/security/prompt-safety.d.ts +100 -0
- package/dist/security/prompt-safety.js +133 -0
- package/dist/server/boot.d.ts +32 -0
- package/dist/server/boot.js +36 -0
- package/dist/server/cli-worker.d.ts +37 -0
- package/dist/server/cli-worker.js +151 -0
- package/dist/server/cli.d.ts +27 -0
- package/dist/server/cli.js +74 -0
- package/dist/server/config.d.ts +42 -0
- package/dist/server/config.js +127 -0
- package/dist/server/health.d.ts +59 -0
- package/dist/server/health.js +90 -0
- package/dist/server/host.d.ts +39 -0
- package/dist/server/host.js +124 -0
- package/dist/server/index.d.ts +15 -0
- package/dist/server/index.js +15 -0
- package/dist/server/main.d.ts +16 -0
- package/dist/server/main.js +31 -0
- package/dist/server/sse-route.d.ts +21 -0
- package/dist/server/sse-route.js +282 -0
- package/dist/skills/index.d.ts +67 -0
- package/dist/skills/index.js +31 -0
- package/dist/skills/resolver.d.ts +54 -0
- package/dist/skills/resolver.js +121 -0
- package/dist/teams/index.d.ts +93 -0
- package/dist/teams/index.js +207 -0
- package/dist/telemetry/index.d.ts +157 -0
- package/dist/telemetry/index.js +71 -0
- package/dist/telemetry/instrument.d.ts +108 -0
- package/dist/telemetry/instrument.js +232 -0
- package/dist/telemetry/log-events.d.ts +17 -0
- package/dist/telemetry/log-events.js +58 -0
- package/dist/telemetry/metrics.d.ts +123 -0
- package/dist/telemetry/metrics.js +135 -0
- package/dist/telemetry/noop.d.ts +39 -0
- package/dist/telemetry/noop.js +143 -0
- package/dist/telemetry/redaction.d.ts +64 -0
- package/dist/telemetry/redaction.js +153 -0
- package/dist/telemetry/spans.d.ts +56 -0
- package/dist/telemetry/spans.js +78 -0
- package/dist/telemetry/trace-context.d.ts +55 -0
- package/dist/telemetry/trace-context.js +60 -0
- package/dist/toolkit/compute.d.ts +53 -0
- package/dist/toolkit/compute.js +152 -0
- package/dist/toolkit/data.d.ts +98 -0
- package/dist/toolkit/data.js +235 -0
- package/dist/toolkit/http.d.ts +113 -0
- package/dist/toolkit/http.js +205 -0
- package/dist/toolkit/index.d.ts +21 -0
- package/dist/toolkit/index.js +17 -0
- package/dist/toolkit/web.d.ts +107 -0
- package/dist/toolkit/web.js +147 -0
- package/dist/tools/define.d.ts +25 -0
- package/dist/tools/define.js +45 -0
- package/dist/tools/delegating.d.ts +132 -0
- package/dist/tools/delegating.js +211 -0
- package/dist/tools/index.d.ts +129 -0
- package/dist/tools/index.js +33 -0
- package/dist/tools/library/compute.d.ts +11 -0
- package/dist/tools/library/compute.js +46 -0
- package/dist/tools/library/data.d.ts +16 -0
- package/dist/tools/library/data.js +92 -0
- package/dist/tools/library/http.d.ts +28 -0
- package/dist/tools/library/http.js +71 -0
- package/dist/tools/library/index.d.ts +97 -0
- package/dist/tools/library/index.js +134 -0
- package/dist/tools/library/knowledge.d.ts +39 -0
- package/dist/tools/library/knowledge.js +58 -0
- package/dist/tools/library/web.d.ts +19 -0
- package/dist/tools/library/web.js +65 -0
- package/dist/tools/meta-tools.d.ts +19 -0
- package/dist/tools/meta-tools.js +36 -0
- package/dist/tools/registry.d.ts +146 -0
- package/dist/tools/registry.js +291 -0
- package/dist/usage/index.d.ts +105 -0
- package/dist/usage/index.js +20 -0
- package/dist/usage/quota.d.ts +258 -0
- package/dist/usage/quota.js +510 -0
- package/dist/usage/recorder.d.ts +29 -0
- package/dist/usage/recorder.js +96 -0
- package/dist/usage/rollups.d.ts +121 -0
- package/dist/usage/rollups.js +157 -0
- package/dist/worker/export.d.ts +57 -0
- package/dist/worker/export.js +81 -0
- package/dist/worker/extraction.d.ts +57 -0
- package/dist/worker/extraction.js +84 -0
- package/dist/worker/main.d.ts +103 -0
- package/dist/worker/main.js +159 -0
- package/package.json +187 -0
|
@@ -0,0 +1,1594 @@
|
|
|
1
|
+
import { EMBEDDING_DIMENSIONS } from "../../persistence/index.js";
|
|
2
|
+
export const MIGRATIONS = [
|
|
3
|
+
{
|
|
4
|
+
id: "0001_conversations",
|
|
5
|
+
up: [
|
|
6
|
+
`CREATE TABLE IF NOT EXISTS conversations (
|
|
7
|
+
tenant_id text NOT NULL,
|
|
8
|
+
id text NOT NULL,
|
|
9
|
+
title text NOT NULL,
|
|
10
|
+
version integer NOT NULL DEFAULT 1,
|
|
11
|
+
archived_at timestamptz,
|
|
12
|
+
deleted_at timestamptz,
|
|
13
|
+
created_at timestamptz NOT NULL,
|
|
14
|
+
updated_at timestamptz NOT NULL,
|
|
15
|
+
PRIMARY KEY (tenant_id, id)
|
|
16
|
+
)`,
|
|
17
|
+
`CREATE INDEX IF NOT EXISTS conversations_tenant_created_idx
|
|
18
|
+
ON conversations (tenant_id, created_at, id) WHERE deleted_at IS NULL`,
|
|
19
|
+
],
|
|
20
|
+
down: [`DROP TABLE IF EXISTS conversations`],
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
// #93 — durable run lifecycle. Columns mirror the `Run` type in `src/runtime/index.ts`; the
|
|
24
|
+
// SPEC's original list named `attempt`/`claimed_at`/`heartbeat_at`, which do not exist on `Run`.
|
|
25
|
+
// The lease is `keepalive_at` + `lease_expires_at`, and retry attempts live in the retry policy
|
|
26
|
+
// rather than the row.
|
|
27
|
+
id: "0002_runs",
|
|
28
|
+
up: [
|
|
29
|
+
`CREATE TABLE IF NOT EXISTS runs (
|
|
30
|
+
tenant_id text NOT NULL,
|
|
31
|
+
id text NOT NULL,
|
|
32
|
+
conversation_id text NOT NULL,
|
|
33
|
+
agent_id text NOT NULL,
|
|
34
|
+
agent_version integer NOT NULL,
|
|
35
|
+
status text NOT NULL,
|
|
36
|
+
created_at timestamptz NOT NULL,
|
|
37
|
+
started_at timestamptz,
|
|
38
|
+
finished_at timestamptz,
|
|
39
|
+
error jsonb,
|
|
40
|
+
claimed_by text,
|
|
41
|
+
keepalive_at timestamptz,
|
|
42
|
+
lease_expires_at timestamptz,
|
|
43
|
+
cancel_requested_at timestamptz,
|
|
44
|
+
PRIMARY KEY (tenant_id, id),
|
|
45
|
+
-- Mirrors RUN_STATUSES (src/runtime/index.ts), which is hyphenated. The GraphQL enum is
|
|
46
|
+
-- underscored; a constraint built from that spelling would reject every waiting-state write.
|
|
47
|
+
CONSTRAINT runs_status_check CHECK (status IN (
|
|
48
|
+
'queued', 'running', 'waiting-for-question', 'waiting-for-approval',
|
|
49
|
+
'retry-pending', 'completed', 'failed', 'cancelled'
|
|
50
|
+
))
|
|
51
|
+
)`,
|
|
52
|
+
// Conversation history, newest-last, stable under concurrent inserts.
|
|
53
|
+
`CREATE INDEX IF NOT EXISTS runs_tenant_conversation_created_idx
|
|
54
|
+
ON runs (tenant_id, conversation_id, created_at, id)`,
|
|
55
|
+
// The reaper sweep. Deliberately NOT tenant-leading: `reapExpired` is cross-tenant by design
|
|
56
|
+
// (a background reaper has no tenant), so a tenant-first index could not serve it. Partial on
|
|
57
|
+
// 'running' because no other status can hold a live lease.
|
|
58
|
+
`CREATE INDEX IF NOT EXISTS runs_running_lease_idx
|
|
59
|
+
ON runs (lease_expires_at) WHERE status = 'running'`,
|
|
60
|
+
],
|
|
61
|
+
down: [
|
|
62
|
+
`DROP INDEX IF EXISTS runs_running_lease_idx`,
|
|
63
|
+
`DROP INDEX IF EXISTS runs_tenant_conversation_created_idx`,
|
|
64
|
+
`DROP TABLE IF EXISTS runs`,
|
|
65
|
+
],
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
// #94 — the durable event log behind streaming catch-up and crash recovery.
|
|
69
|
+
//
|
|
70
|
+
// The composite primary key is the load-bearing constraint: it makes a duplicate sequence
|
|
71
|
+
// impossible, which is what turns `append` into the idempotent no-op the port promises (via
|
|
72
|
+
// ON CONFLICT DO NOTHING). It does *not* make gaps impossible — gapless numbering is the
|
|
73
|
+
// emitter's contract, and no schema can compel it.
|
|
74
|
+
//
|
|
75
|
+
// No conversation column: `RunEvent` carries no `conversationId` (see `EventBase` in
|
|
76
|
+
// src/core/events.ts), so the conversation-level index the SPEC suggested cannot be built.
|
|
77
|
+
// Replay is per-run and the primary key serves it directly.
|
|
78
|
+
id: "0003_run_events",
|
|
79
|
+
up: [
|
|
80
|
+
`CREATE TABLE IF NOT EXISTS run_events (
|
|
81
|
+
tenant_id text NOT NULL,
|
|
82
|
+
run_id text NOT NULL,
|
|
83
|
+
sequence integer NOT NULL,
|
|
84
|
+
type text NOT NULL,
|
|
85
|
+
event jsonb NOT NULL,
|
|
86
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
87
|
+
PRIMARY KEY (tenant_id, run_id, sequence),
|
|
88
|
+
CONSTRAINT run_events_sequence_positive CHECK (sequence > 0)
|
|
89
|
+
)`,
|
|
90
|
+
],
|
|
91
|
+
down: [`DROP TABLE IF EXISTS run_events`],
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
// #95 — the resume point a recovered worker restarts from.
|
|
95
|
+
//
|
|
96
|
+
// Keyed per run, NOT per sequence. The port documents `save` as overwriting the run's checkpoint
|
|
97
|
+
// and `latest(runId)` is the only read, so a (tenant_id, run_id, sequence) key would store a row
|
|
98
|
+
// per agent-loop step on a table nothing reads historically — run_events-shaped growth with no
|
|
99
|
+
// reader. One slot per run, upserted, matching the reference adapter.
|
|
100
|
+
//
|
|
101
|
+
// `step` is the agent-loop index (an integer bounded by ExecutionLimits.maxSteps), not text.
|
|
102
|
+
//
|
|
103
|
+
// ON DELETE CASCADE: deleting a run must not be able to leave an orphan checkpoint, and nothing
|
|
104
|
+
// that deletes a run should have to know checkpoints exist. RESTRICT would make run deletion
|
|
105
|
+
// fail while a checkpoint lives, which is strictly worse.
|
|
106
|
+
id: "0004_checkpoints",
|
|
107
|
+
up: [
|
|
108
|
+
`CREATE TABLE IF NOT EXISTS checkpoints (
|
|
109
|
+
tenant_id text NOT NULL,
|
|
110
|
+
run_id text NOT NULL,
|
|
111
|
+
sequence integer NOT NULL,
|
|
112
|
+
step integer NOT NULL,
|
|
113
|
+
state jsonb NOT NULL,
|
|
114
|
+
updated_at timestamptz NOT NULL,
|
|
115
|
+
PRIMARY KEY (tenant_id, run_id),
|
|
116
|
+
CONSTRAINT checkpoints_run_fk
|
|
117
|
+
FOREIGN KEY (tenant_id, run_id) REFERENCES runs (tenant_id, id) ON DELETE CASCADE,
|
|
118
|
+
CONSTRAINT checkpoints_counters_non_negative CHECK (sequence >= 0 AND step >= 0)
|
|
119
|
+
)`,
|
|
120
|
+
],
|
|
121
|
+
down: [`DROP TABLE IF EXISTS checkpoints`],
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
// #96 — the conversation itself: its messages, the agent manifests, and the binding that says
|
|
125
|
+
// which agent version owns a thread.
|
|
126
|
+
id: "0005_messages_agents",
|
|
127
|
+
up: [
|
|
128
|
+
`CREATE TABLE IF NOT EXISTS messages (
|
|
129
|
+
tenant_id text NOT NULL,
|
|
130
|
+
id text NOT NULL,
|
|
131
|
+
conversation_id text NOT NULL,
|
|
132
|
+
run_id text,
|
|
133
|
+
role text NOT NULL,
|
|
134
|
+
parts jsonb NOT NULL,
|
|
135
|
+
created_at timestamptz NOT NULL,
|
|
136
|
+
PRIMARY KEY (tenant_id, id),
|
|
137
|
+
CONSTRAINT messages_conversation_fk
|
|
138
|
+
FOREIGN KEY (tenant_id, conversation_id) REFERENCES conversations (tenant_id, id)
|
|
139
|
+
ON DELETE CASCADE
|
|
140
|
+
)`,
|
|
141
|
+
// The composite ordering the stable cursor pages on. `id` breaks ties, which is what makes
|
|
142
|
+
// paging safe under concurrent inserts sharing a created_at — a timestamp alone would let a
|
|
143
|
+
// row slip between pages.
|
|
144
|
+
`CREATE INDEX IF NOT EXISTS messages_tenant_conversation_created_idx
|
|
145
|
+
ON messages (tenant_id, conversation_id, created_at, id)`,
|
|
146
|
+
// One row per version: a thread pinned to v1 must still resolve v1 after v2 is registered.
|
|
147
|
+
// Tenant-leading — this is the store whose in-memory version leaked across tenants (#91).
|
|
148
|
+
`CREATE TABLE IF NOT EXISTS agents (
|
|
149
|
+
tenant_id text NOT NULL,
|
|
150
|
+
id text NOT NULL,
|
|
151
|
+
version integer NOT NULL,
|
|
152
|
+
manifest jsonb NOT NULL,
|
|
153
|
+
created_at timestamptz NOT NULL,
|
|
154
|
+
updated_at timestamptz NOT NULL,
|
|
155
|
+
PRIMARY KEY (tenant_id, id, version),
|
|
156
|
+
CONSTRAINT agents_version_positive CHECK (version > 0)
|
|
157
|
+
)`,
|
|
158
|
+
// agent_version_policy was absent from the SPEC. Without it a binding cannot express what it
|
|
159
|
+
// exists to express, and agent_version is NULL for a 'latest' binding — hence nullable, with a
|
|
160
|
+
// constraint tying the two together instead of leaving the pair free to contradict itself.
|
|
161
|
+
//
|
|
162
|
+
// No foreign key to `agents`: a 'latest' binding carries no version, so a composite
|
|
163
|
+
// (agent_id, agent_version) reference cannot be enforced for it, and binding to an agent whose
|
|
164
|
+
// manifest is not yet registered is legitimate.
|
|
165
|
+
`CREATE TABLE IF NOT EXISTS conversation_bindings (
|
|
166
|
+
tenant_id text NOT NULL,
|
|
167
|
+
conversation_id text NOT NULL,
|
|
168
|
+
agent_id text NOT NULL,
|
|
169
|
+
agent_version_policy text NOT NULL,
|
|
170
|
+
agent_version integer,
|
|
171
|
+
bound_at timestamptz NOT NULL,
|
|
172
|
+
PRIMARY KEY (tenant_id, conversation_id),
|
|
173
|
+
CONSTRAINT conversation_bindings_conversation_fk
|
|
174
|
+
FOREIGN KEY (tenant_id, conversation_id) REFERENCES conversations (tenant_id, id)
|
|
175
|
+
ON DELETE CASCADE,
|
|
176
|
+
CONSTRAINT conversation_bindings_policy_check
|
|
177
|
+
CHECK (agent_version_policy IN ('pinned', 'latest')),
|
|
178
|
+
CONSTRAINT conversation_bindings_pinned_has_version
|
|
179
|
+
CHECK ((agent_version_policy = 'pinned' AND agent_version IS NOT NULL)
|
|
180
|
+
OR (agent_version_policy = 'latest'))
|
|
181
|
+
)`,
|
|
182
|
+
],
|
|
183
|
+
down: [
|
|
184
|
+
`DROP TABLE IF EXISTS conversation_bindings`,
|
|
185
|
+
`DROP TABLE IF EXISTS agents`,
|
|
186
|
+
`DROP INDEX IF EXISTS messages_tenant_conversation_created_idx`,
|
|
187
|
+
`DROP TABLE IF EXISTS messages`,
|
|
188
|
+
],
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
// #97 — cross-turn working memory and compacted history.
|
|
192
|
+
id: "0006_session_state",
|
|
193
|
+
up: [
|
|
194
|
+
// One row per conversation, guarded by `version`. The version is the whole point: two runs on
|
|
195
|
+
// one conversation must not interleave into a lost update, and the compare-and-set happens in
|
|
196
|
+
// the UPDATE's WHERE clause rather than in the caller.
|
|
197
|
+
`CREATE TABLE IF NOT EXISTS session_state (
|
|
198
|
+
tenant_id text NOT NULL,
|
|
199
|
+
conversation_id text NOT NULL,
|
|
200
|
+
state jsonb NOT NULL,
|
|
201
|
+
version integer NOT NULL,
|
|
202
|
+
updated_at timestamptz NOT NULL,
|
|
203
|
+
PRIMARY KEY (tenant_id, conversation_id),
|
|
204
|
+
CONSTRAINT session_state_conversation_fk
|
|
205
|
+
FOREIGN KEY (tenant_id, conversation_id) REFERENCES conversations (tenant_id, id)
|
|
206
|
+
ON DELETE CASCADE,
|
|
207
|
+
CONSTRAINT session_state_version_positive CHECK (version > 0)
|
|
208
|
+
)`,
|
|
209
|
+
// Corrected against `ThreadSummary` (src/persistence/index.ts). The SPEC had
|
|
210
|
+
// `covers_up_to timestamptz`, `summary jsonb` and a `token_estimate` column:
|
|
211
|
+
// - the field is `coversUpToMessageId`, a message id — a summary covers history up to a
|
|
212
|
+
// specific *message*, which is what lets the assembler keep everything after it verbatim.
|
|
213
|
+
// A timestamp cannot identify that boundary.
|
|
214
|
+
// - `summary` is a string, so text.
|
|
215
|
+
// - nothing on the type could populate `token_estimate`.
|
|
216
|
+
`CREATE TABLE IF NOT EXISTS thread_summaries (
|
|
217
|
+
tenant_id text NOT NULL,
|
|
218
|
+
conversation_id text NOT NULL,
|
|
219
|
+
version integer NOT NULL,
|
|
220
|
+
summary text NOT NULL,
|
|
221
|
+
covers_up_to_message_id text NOT NULL,
|
|
222
|
+
created_at timestamptz NOT NULL,
|
|
223
|
+
PRIMARY KEY (tenant_id, conversation_id, version),
|
|
224
|
+
CONSTRAINT thread_summaries_conversation_fk
|
|
225
|
+
FOREIGN KEY (tenant_id, conversation_id) REFERENCES conversations (tenant_id, id)
|
|
226
|
+
ON DELETE CASCADE,
|
|
227
|
+
CONSTRAINT thread_summaries_version_positive CHECK (version > 0)
|
|
228
|
+
)`,
|
|
229
|
+
],
|
|
230
|
+
down: [`DROP TABLE IF EXISTS thread_summaries`, `DROP TABLE IF EXISTS session_state`],
|
|
231
|
+
},
|
|
232
|
+
{
|
|
233
|
+
// #98 — per-conversation run serialization.
|
|
234
|
+
//
|
|
235
|
+
// A slot table rather than advisory locks, and the reasoning is not close. An advisory xact lock
|
|
236
|
+
// lives exactly as long as its transaction, but the active slot must live as long as the *run* —
|
|
237
|
+
// which includes `waiting-for-question` and `waiting-for-approval`, states that exist to wait for
|
|
238
|
+
// a human and can last hours. Holding a transaction open that long pins a connection and blocks
|
|
239
|
+
// VACUUM across the database. Worse, the port promises a FIFO `position` and a `depth`, and a
|
|
240
|
+
// lock has no ordering, no membership and no introspection — the queue has to live in a row
|
|
241
|
+
// regardless, at which point the lock adds nothing. Postgres advisory locks are also not FIFO.
|
|
242
|
+
id: "0007_run_coordination",
|
|
243
|
+
up: [
|
|
244
|
+
`CREATE TABLE IF NOT EXISTS conversation_run_slots (
|
|
245
|
+
tenant_id text NOT NULL,
|
|
246
|
+
conversation_id text NOT NULL,
|
|
247
|
+
-- Nullable: an idle conversation has a row with no holder. Deliberately NOT a foreign key to
|
|
248
|
+
-- the runs table: claimOrEnqueue is legitimately called before the run row is committed, so the
|
|
249
|
+
-- reference cannot be enforced without breaking the caller it exists to serve.
|
|
250
|
+
active_run_id text,
|
|
251
|
+
queued jsonb NOT NULL DEFAULT '[]'::jsonb,
|
|
252
|
+
updated_at timestamptz NOT NULL,
|
|
253
|
+
PRIMARY KEY (tenant_id, conversation_id),
|
|
254
|
+
CONSTRAINT conversation_run_slots_conversation_fk
|
|
255
|
+
FOREIGN KEY (tenant_id, conversation_id) REFERENCES conversations (tenant_id, id)
|
|
256
|
+
ON DELETE CASCADE,
|
|
257
|
+
-- The queue is an ordered array. Without this a malformed write could store an object and
|
|
258
|
+
-- every position/depth answer afterwards would be silently wrong.
|
|
259
|
+
CONSTRAINT conversation_run_slots_queued_is_array CHECK (jsonb_typeof(queued) = 'array')
|
|
260
|
+
)`,
|
|
261
|
+
],
|
|
262
|
+
down: [`DROP TABLE IF EXISTS conversation_run_slots`],
|
|
263
|
+
},
|
|
264
|
+
{
|
|
265
|
+
// #99 — durable questions, approvals and standing grants.
|
|
266
|
+
//
|
|
267
|
+
// Two interaction tables, not the one the SPEC proposed. PendingQuestion and PendingApproval
|
|
268
|
+
// (src/hitl/index.ts) share three fields; six of the approval's need to be real columns or
|
|
269
|
+
// constraints, idempotency_key above all, since the replay guarantee rests on a unique index over
|
|
270
|
+
// it. A single table with a jsonb payload could not carry that constraint.
|
|
271
|
+
//
|
|
272
|
+
// No conversation_id on either: neither type has one (they are scoped by tenant + run), the same
|
|
273
|
+
// correction run_events needed in #94.
|
|
274
|
+
//
|
|
275
|
+
// No status column either. A question is pending exactly when answered_at IS NULL. A stored
|
|
276
|
+
// status is a second source of truth for one fact, and it fails silently — a row marked pending
|
|
277
|
+
// with an answer already recorded. The partial indexes below serve the waiting-run lookup the
|
|
278
|
+
// status column was meant for, and stay the size of the backlog rather than of all history.
|
|
279
|
+
id: "0008_hitl",
|
|
280
|
+
up: [
|
|
281
|
+
`CREATE TABLE IF NOT EXISTS interaction_questions (
|
|
282
|
+
tenant_id text NOT NULL,
|
|
283
|
+
id text NOT NULL,
|
|
284
|
+
run_id text NOT NULL,
|
|
285
|
+
questions jsonb NOT NULL,
|
|
286
|
+
created_at timestamptz NOT NULL,
|
|
287
|
+
answered_at timestamptz,
|
|
288
|
+
answers jsonb,
|
|
289
|
+
PRIMARY KEY (tenant_id, id),
|
|
290
|
+
CONSTRAINT interaction_questions_run_fk
|
|
291
|
+
FOREIGN KEY (tenant_id, run_id) REFERENCES runs (tenant_id, id) ON DELETE CASCADE,
|
|
292
|
+
-- Answered means both fields, or neither. A half-written resolution would read as answered
|
|
293
|
+
-- while carrying no answer, and the resuming run would execute on nothing.
|
|
294
|
+
CONSTRAINT interaction_questions_answer_complete CHECK (
|
|
295
|
+
(answered_at IS NULL AND answers IS NULL) OR (answered_at IS NOT NULL AND answers IS NOT NULL)
|
|
296
|
+
)
|
|
297
|
+
)`,
|
|
298
|
+
// Partial, and deliberately not unique: findPendingQuestion returns a single question, so at
|
|
299
|
+
// most one pending per run is plausibly the real invariant — but the port does not say so and
|
|
300
|
+
// the reference adapter permits several. Enforcing it here would be stricter than the contract.
|
|
301
|
+
// Flagged as an open question on #99 rather than decided unilaterally in an index.
|
|
302
|
+
`CREATE INDEX IF NOT EXISTS interaction_questions_pending_run_idx
|
|
303
|
+
ON interaction_questions (tenant_id, run_id) WHERE answered_at IS NULL`,
|
|
304
|
+
`CREATE TABLE IF NOT EXISTS interaction_approvals (
|
|
305
|
+
tenant_id text NOT NULL,
|
|
306
|
+
id text NOT NULL,
|
|
307
|
+
run_id text NOT NULL,
|
|
308
|
+
tool_name text NOT NULL,
|
|
309
|
+
normalized_input jsonb NOT NULL,
|
|
310
|
+
risk_category text NOT NULL,
|
|
311
|
+
summary text NOT NULL,
|
|
312
|
+
estimated_cost_minor_units integer,
|
|
313
|
+
expires_at timestamptz NOT NULL,
|
|
314
|
+
idempotency_key text NOT NULL,
|
|
315
|
+
decided_at timestamptz,
|
|
316
|
+
decision text,
|
|
317
|
+
PRIMARY KEY (tenant_id, id),
|
|
318
|
+
CONSTRAINT interaction_approvals_run_fk
|
|
319
|
+
FOREIGN KEY (tenant_id, run_id) REFERENCES runs (tenant_id, id) ON DELETE CASCADE,
|
|
320
|
+
CONSTRAINT interaction_approvals_decision_complete CHECK (
|
|
321
|
+
(decided_at IS NULL AND decision IS NULL) OR (decided_at IS NOT NULL AND decision IS NOT NULL)
|
|
322
|
+
),
|
|
323
|
+
-- Mirrors APPROVAL_DECISIONS (src/hitl/index.ts).
|
|
324
|
+
CONSTRAINT interaction_approvals_decision_check CHECK (
|
|
325
|
+
decision IS NULL OR decision IN ('allow-once', 'allow-conversation', 'allow-always', 'deny')
|
|
326
|
+
)
|
|
327
|
+
)`,
|
|
328
|
+
// The enforcement point for AC-2. A replayed approval must not be able to authorise the same
|
|
329
|
+
// side effect twice, and the database is the only place that holds across processes -- an
|
|
330
|
+
// in-process check cannot. The SPEC put this index on approval_grants, where idempotency_key
|
|
331
|
+
// does not exist, which would have left the guarantee unenforced while looking implemented.
|
|
332
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS interaction_approvals_idempotency_idx
|
|
333
|
+
ON interaction_approvals (tenant_id, idempotency_key)`,
|
|
334
|
+
`CREATE INDEX IF NOT EXISTS interaction_approvals_pending_run_idx
|
|
335
|
+
ON interaction_approvals (tenant_id, run_id) WHERE decided_at IS NULL`,
|
|
336
|
+
// Standing grants. No run_id, effect, request, decision, decided_by, decided_at or
|
|
337
|
+
// idempotency_key: none of them exist on ApprovalGrant. A grant is a standing permission, not a
|
|
338
|
+
// decision record -- the decision lives on the approval.
|
|
339
|
+
`CREATE TABLE IF NOT EXISTS approval_grants (
|
|
340
|
+
tenant_id text NOT NULL,
|
|
341
|
+
id text NOT NULL,
|
|
342
|
+
scope text NOT NULL,
|
|
343
|
+
tool_name_or_category text NOT NULL,
|
|
344
|
+
conversation_id text,
|
|
345
|
+
granted_at timestamptz NOT NULL,
|
|
346
|
+
expires_at timestamptz,
|
|
347
|
+
revoked_at timestamptz,
|
|
348
|
+
PRIMARY KEY (tenant_id, id),
|
|
349
|
+
-- Mirrors ApprovalScope (src/hitl/index.ts).
|
|
350
|
+
CONSTRAINT approval_grants_scope_check
|
|
351
|
+
CHECK (scope IN ('principal', 'tenant', 'category', 'conversation')),
|
|
352
|
+
-- A conversation-scoped grant with no conversation could match nothing at best, and at worst
|
|
353
|
+
-- a query that forgot the scope check would treat it as tenant-wide -- turning a
|
|
354
|
+
-- one-conversation approval into a standing one, which the port forbids by name.
|
|
355
|
+
CONSTRAINT approval_grants_conversation_scope_has_conversation
|
|
356
|
+
CHECK (scope <> 'conversation' OR conversation_id IS NOT NULL)
|
|
357
|
+
)`,
|
|
358
|
+
`CREATE INDEX IF NOT EXISTS approval_grants_active_idx
|
|
359
|
+
ON approval_grants (tenant_id, tool_name_or_category) WHERE revoked_at IS NULL`,
|
|
360
|
+
],
|
|
361
|
+
down: [
|
|
362
|
+
`DROP TABLE IF EXISTS approval_grants`,
|
|
363
|
+
`DROP TABLE IF EXISTS interaction_approvals`,
|
|
364
|
+
`DROP TABLE IF EXISTS interaction_questions`,
|
|
365
|
+
],
|
|
366
|
+
},
|
|
367
|
+
{
|
|
368
|
+
// #100 — the cost ledger and the replay guard.
|
|
369
|
+
//
|
|
370
|
+
// Corrected against UsageEvent (src/usage/index.ts). The SPEC proposed `cost_micros bigint`; the
|
|
371
|
+
// field is `costMinorUnits`, documented as integer minor units of the tenant's accounting
|
|
372
|
+
// currency. Minor units are 10^-2, micros are 10^-6, so that column name sets up a 10000x error
|
|
373
|
+
// in precisely the rollups the SPEC says depend on it being exact.
|
|
374
|
+
//
|
|
375
|
+
// It also omitted `currency`, which is required and without which a minor-unit integer means
|
|
376
|
+
// nothing -- 250 is EUR 2.50 or JPY 250 -- so a rollup summing across currencies would be
|
|
377
|
+
// confidently wrong. And it omitted `step_id`, which is load-bearing: appends are idempotent on
|
|
378
|
+
// (runId, stepId), so without it a recovered run double-counts.
|
|
379
|
+
//
|
|
380
|
+
// `principal_id` was dropped: no such field exists on the type.
|
|
381
|
+
id: "0009_usage_idempotency",
|
|
382
|
+
up: [
|
|
383
|
+
`CREATE TABLE IF NOT EXISTS usage_records (
|
|
384
|
+
tenant_id text NOT NULL,
|
|
385
|
+
id text NOT NULL,
|
|
386
|
+
-- Computed by usageDedupeKey (src/usage/index.ts), which both adapters import, so the
|
|
387
|
+
-- idempotency rule cannot drift between them. Stored rather than expressed as an index on
|
|
388
|
+
-- COALESCE so the rule lives in one function instead of in one function and one index.
|
|
389
|
+
dedupe_key text NOT NULL,
|
|
390
|
+
run_id text NOT NULL,
|
|
391
|
+
conversation_id text,
|
|
392
|
+
step_id text,
|
|
393
|
+
tool_call_id text,
|
|
394
|
+
model_id text NOT NULL,
|
|
395
|
+
input_tokens integer NOT NULL,
|
|
396
|
+
output_tokens integer NOT NULL,
|
|
397
|
+
cached_input_tokens integer NOT NULL,
|
|
398
|
+
reasoning_tokens integer,
|
|
399
|
+
cost_minor_units integer NOT NULL,
|
|
400
|
+
currency text NOT NULL,
|
|
401
|
+
occurred_at timestamptz NOT NULL,
|
|
402
|
+
PRIMARY KEY (tenant_id, id),
|
|
403
|
+
CONSTRAINT usage_records_run_fk
|
|
404
|
+
FOREIGN KEY (tenant_id, run_id) REFERENCES runs (tenant_id, id) ON DELETE CASCADE,
|
|
405
|
+
-- Integer columns are what make AC-3 true. A float column would sum with drift, and the
|
|
406
|
+
-- drift would land in an invoice.
|
|
407
|
+
CONSTRAINT usage_records_non_negative CHECK (
|
|
408
|
+
input_tokens >= 0 AND output_tokens >= 0 AND cached_input_tokens >= 0
|
|
409
|
+
AND (reasoning_tokens IS NULL OR reasoning_tokens >= 0) AND cost_minor_units >= 0
|
|
410
|
+
)
|
|
411
|
+
)`,
|
|
412
|
+
// The append-idempotency guarantee. A recovered run that re-records a step it already logged
|
|
413
|
+
// must be a no-op, because the alternative is double-billing.
|
|
414
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS usage_records_dedupe_idx
|
|
415
|
+
ON usage_records (tenant_id, dedupe_key)`,
|
|
416
|
+
// Run listing, keyset-paged. `id` breaks ties so a page boundary cannot drop or repeat a row.
|
|
417
|
+
`CREATE INDEX IF NOT EXISTS usage_records_run_occurred_idx
|
|
418
|
+
ON usage_records (tenant_id, run_id, occurred_at, id)`,
|
|
419
|
+
// The rollup index REQ-031 will need. Tenant-leading and time-ordered.
|
|
420
|
+
`CREATE INDEX IF NOT EXISTS usage_records_tenant_occurred_idx
|
|
421
|
+
ON usage_records (tenant_id, occurred_at, id)`,
|
|
422
|
+
// No `scope` column: nothing in IdempotencyStore has one. No `expires_at` either -- `put`
|
|
423
|
+
// takes no TTL and the port has no prune method, so nothing could populate it and an
|
|
424
|
+
// always-NULL expiry column would read as a retention policy that does not exist. Retention is
|
|
425
|
+
// flagged as an open question on #100; `created_at` plus its index is the mechanism a prune
|
|
426
|
+
// needs, which is what AC-5 actually requires.
|
|
427
|
+
`CREATE TABLE IF NOT EXISTS idempotency_keys (
|
|
428
|
+
tenant_id text NOT NULL,
|
|
429
|
+
key text NOT NULL,
|
|
430
|
+
result jsonb NOT NULL,
|
|
431
|
+
created_at timestamptz NOT NULL,
|
|
432
|
+
PRIMARY KEY (tenant_id, key)
|
|
433
|
+
)`,
|
|
434
|
+
`CREATE INDEX IF NOT EXISTS idempotency_keys_created_idx ON idempotency_keys (created_at)`,
|
|
435
|
+
],
|
|
436
|
+
down: [`DROP TABLE IF EXISTS idempotency_keys`, `DROP TABLE IF EXISTS usage_records`],
|
|
437
|
+
},
|
|
438
|
+
{
|
|
439
|
+
// #101 — tenant configuration: versioned skills and registered outbound MCP servers.
|
|
440
|
+
//
|
|
441
|
+
// Corrected against SkillVersion (src/skills/index.ts) and McpServerConnection (src/mcp/index.ts).
|
|
442
|
+
// The SPEC named `content` for `instructions`, omitted `description` -- which is the field
|
|
443
|
+
// discovery actually puts in model context, so listCatalog could not return a usable entry
|
|
444
|
+
// without it -- omitted `source`, and proposed `enabled boolean` for a three-state `status`
|
|
445
|
+
// (draft/active/archived) that a boolean cannot represent.
|
|
446
|
+
//
|
|
447
|
+
// The primary key is on (tenant_id, name, version), not (tenant_id, id, version): findVersion
|
|
448
|
+
// looks up by name and version, and the reference adapter keys on name@version.
|
|
449
|
+
id: "0010_skills_mcp",
|
|
450
|
+
up: [
|
|
451
|
+
`CREATE TABLE IF NOT EXISTS skills (
|
|
452
|
+
tenant_id text NOT NULL,
|
|
453
|
+
id text NOT NULL,
|
|
454
|
+
name text NOT NULL,
|
|
455
|
+
description text NOT NULL,
|
|
456
|
+
source text NOT NULL,
|
|
457
|
+
version integer NOT NULL,
|
|
458
|
+
instructions text NOT NULL,
|
|
459
|
+
status text NOT NULL,
|
|
460
|
+
created_at timestamptz NOT NULL,
|
|
461
|
+
created_by text,
|
|
462
|
+
PRIMARY KEY (tenant_id, name, version),
|
|
463
|
+
CONSTRAINT skills_id_unique UNIQUE (tenant_id, id),
|
|
464
|
+
-- Mirrors SKILL_SOURCES and SkillStatus.
|
|
465
|
+
CONSTRAINT skills_source_check CHECK (source IN ('built-in', 'tenant', 'plugin')),
|
|
466
|
+
CONSTRAINT skills_status_check CHECK (status IN ('draft', 'active', 'archived')),
|
|
467
|
+
CONSTRAINT skills_version_positive CHECK (version > 0),
|
|
468
|
+
-- Mirrors SKILL_LIMITS. The store already runs validateSkillInput, so these exist for the
|
|
469
|
+
-- path that bypasses it: a migration, a data fix, anything writing SQL directly. A limit
|
|
470
|
+
-- enforced only in application code is a limit that holds until someone opens psql.
|
|
471
|
+
CONSTRAINT skills_name_slug CHECK (
|
|
472
|
+
name ~ '^[a-z0-9]+(-[a-z0-9]+)*$' AND length(name) <= 64
|
|
473
|
+
),
|
|
474
|
+
CONSTRAINT skills_description_bounds CHECK (length(description) BETWEEN 20 AND 1024),
|
|
475
|
+
CONSTRAINT skills_instructions_bounds CHECK (length(instructions) <= 20000)
|
|
476
|
+
)`,
|
|
477
|
+
// Discovery reads the latest *active* version per name, so the index is partial on active and
|
|
478
|
+
// descends by version -- the shape that answers the catalog query without a sort.
|
|
479
|
+
`CREATE INDEX IF NOT EXISTS skills_active_catalog_idx
|
|
480
|
+
ON skills (tenant_id, name, version DESC) WHERE status = 'active'`,
|
|
481
|
+
// No egress_policy column: the egress policy is a parameter of the *store*
|
|
482
|
+
// (createMemoryMcpConnectionStore(egress)) -- a deployment-level rule, not per-row data.
|
|
483
|
+
// Nothing could populate it. No updated_at either; the type has no such field.
|
|
484
|
+
`CREATE TABLE IF NOT EXISTS mcp_connections (
|
|
485
|
+
tenant_id text NOT NULL,
|
|
486
|
+
id text NOT NULL,
|
|
487
|
+
label text NOT NULL,
|
|
488
|
+
transport text NOT NULL,
|
|
489
|
+
endpoint text NOT NULL,
|
|
490
|
+
auth_kind text NOT NULL,
|
|
491
|
+
-- A reference, never a value. McpAuth has no field capable of holding a secret and neither
|
|
492
|
+
-- does this table -- which is a stronger guarantee than a pattern trying to recognise one.
|
|
493
|
+
auth_credential_ref text,
|
|
494
|
+
enabled boolean NOT NULL,
|
|
495
|
+
created_at timestamptz NOT NULL,
|
|
496
|
+
last_handshake_at timestamptz,
|
|
497
|
+
last_error text,
|
|
498
|
+
PRIMARY KEY (tenant_id, id),
|
|
499
|
+
-- Mirrors MCP_TRANSPORTS.
|
|
500
|
+
CONSTRAINT mcp_connections_transport_check
|
|
501
|
+
CHECK (transport IN ('stdio', 'streamable-http', 'sse')),
|
|
502
|
+
CONSTRAINT mcp_connections_auth_kind_check
|
|
503
|
+
CHECK (auth_kind IN ('none', 'bearer', 'oauth')),
|
|
504
|
+
-- The discriminated union's shape, enforced by the database rather than trusted: 'none'
|
|
505
|
+
-- carries no reference, and the two that need one cannot be stored without it. Half a
|
|
506
|
+
-- bearer connection would fail at handshake time, far from the write that caused it.
|
|
507
|
+
CONSTRAINT mcp_connections_auth_ref_pairing CHECK (
|
|
508
|
+
(auth_kind = 'none' AND auth_credential_ref IS NULL)
|
|
509
|
+
OR (auth_kind IN ('bearer', 'oauth') AND auth_credential_ref IS NOT NULL)
|
|
510
|
+
)
|
|
511
|
+
)`,
|
|
512
|
+
`CREATE INDEX IF NOT EXISTS mcp_connections_enabled_idx
|
|
513
|
+
ON mcp_connections (tenant_id, id) WHERE enabled`,
|
|
514
|
+
],
|
|
515
|
+
down: [`DROP TABLE IF EXISTS mcp_connections`, `DROP TABLE IF EXISTS skills`],
|
|
516
|
+
},
|
|
517
|
+
{
|
|
518
|
+
// #102 — per-user memory and the tool-output offload. Completes the Postgres column.
|
|
519
|
+
//
|
|
520
|
+
// Corrected against PrincipalMemoryEntry. The SPEC had `content text, source text` and nothing
|
|
521
|
+
// else: `content` is `text`, `source` has no field to populate it, and it omitted `tags`,
|
|
522
|
+
// `salience`, `version` and `disabled_at` -- all four load-bearing. Without `salience` retrieve
|
|
523
|
+
// cannot order "most salient first"; without `version` the optimistic-concurrency guard on
|
|
524
|
+
// update cannot exist; and without `disabled_at` there is nowhere to record that a user switched
|
|
525
|
+
// a memory off, so a disabled memory would keep influencing every turn.
|
|
526
|
+
id: "0011_principal_memory_blobs",
|
|
527
|
+
up: [
|
|
528
|
+
`CREATE TABLE IF NOT EXISTS principal_memory (
|
|
529
|
+
tenant_id text NOT NULL,
|
|
530
|
+
principal_id text NOT NULL,
|
|
531
|
+
id text NOT NULL,
|
|
532
|
+
text text NOT NULL,
|
|
533
|
+
tags jsonb NOT NULL DEFAULT '[]'::jsonb,
|
|
534
|
+
salience integer NOT NULL,
|
|
535
|
+
version integer NOT NULL,
|
|
536
|
+
created_at timestamptz NOT NULL,
|
|
537
|
+
updated_at timestamptz NOT NULL,
|
|
538
|
+
disabled_at timestamptz,
|
|
539
|
+
-- Principal-leading, so scoping is structural rather than a filter someone can forget. The
|
|
540
|
+
-- SPEC got this right and the reasoning is worth keeping: there is no key shape here that
|
|
541
|
+
-- permits a cross-principal read by accident.
|
|
542
|
+
PRIMARY KEY (tenant_id, principal_id, id),
|
|
543
|
+
CONSTRAINT principal_memory_version_positive CHECK (version > 0),
|
|
544
|
+
CONSTRAINT principal_memory_tags_is_array CHECK (jsonb_typeof(tags) = 'array'),
|
|
545
|
+
-- Mirrors MEMORY_LIMITS (src/principal-memory/index.ts). Enforced here as well as in the
|
|
546
|
+
-- extraction gate, because the gate is application code and this is not.
|
|
547
|
+
CONSTRAINT principal_memory_text_bounds CHECK (length(text) BETWEEN 1 AND 1000),
|
|
548
|
+
CONSTRAINT principal_memory_tag_count CHECK (jsonb_array_length(tags) <= 8)
|
|
549
|
+
)`,
|
|
550
|
+
// list() pages by (created_at, id).
|
|
551
|
+
`CREATE INDEX IF NOT EXISTS principal_memory_list_idx
|
|
552
|
+
ON principal_memory (tenant_id, principal_id, created_at, id)`,
|
|
553
|
+
// retrieve() reads active entries, most salient first. Partial on active because a disabled
|
|
554
|
+
// entry must never be a candidate -- the index cannot return what it does not contain.
|
|
555
|
+
`CREATE INDEX IF NOT EXISTS principal_memory_retrieve_idx
|
|
556
|
+
ON principal_memory (tenant_id, principal_id, salience DESC) WHERE disabled_at IS NULL`,
|
|
557
|
+
// The ref generator. A sequence rather than a process-local counter: the reference adapter's
|
|
558
|
+
// counter resets with the process, which for a durable store would hand out a ref that already
|
|
559
|
+
// belongs to someone else's value after a restart.
|
|
560
|
+
`CREATE SEQUENCE IF NOT EXISTS blob_ref_seq`,
|
|
561
|
+
// The value itself, not a pointer. BlobStore is `put(value) -> ref` / `get(ref) -> value`, so a
|
|
562
|
+
// metadata-and-pointer row could not serve get() at all -- there is nothing in the port to
|
|
563
|
+
// fetch bytes with. The metadata table the SPEC described belongs to FileMetadataStore (#129)
|
|
564
|
+
// or ArtifactStore (#133), which exist as deliberate placeholders.
|
|
565
|
+
`CREATE TABLE IF NOT EXISTS blobs (
|
|
566
|
+
tenant_id text NOT NULL,
|
|
567
|
+
ref text NOT NULL,
|
|
568
|
+
value jsonb NOT NULL,
|
|
569
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
570
|
+
PRIMARY KEY (tenant_id, ref)
|
|
571
|
+
)`,
|
|
572
|
+
],
|
|
573
|
+
down: [
|
|
574
|
+
`DROP TABLE IF EXISTS blobs`,
|
|
575
|
+
`DROP SEQUENCE IF EXISTS blob_ref_seq`,
|
|
576
|
+
`DROP TABLE IF EXISTS principal_memory`,
|
|
577
|
+
],
|
|
578
|
+
},
|
|
579
|
+
{
|
|
580
|
+
// The claim that makes `allow-once` mean once (docs/04 -> How the loop closes).
|
|
581
|
+
//
|
|
582
|
+
// `decided_at` alone cannot express "approved and already executed". Without a second column the
|
|
583
|
+
// resumed run has no way to tell an approval it still owes an execution from one it has already
|
|
584
|
+
// performed, so either it never runs the approved call or it runs it on every resumption. A grant
|
|
585
|
+
// was the other option and the wrong one: a grant is standing, and issuing one for a one-time
|
|
586
|
+
// decision widens the authority the human gave.
|
|
587
|
+
//
|
|
588
|
+
// The claim lives in the database rather than in the runtime because two workers can race a
|
|
589
|
+
// resumed run, and an in-process check cannot make one of them lose.
|
|
590
|
+
id: "0012_approval_consumption",
|
|
591
|
+
up: [
|
|
592
|
+
`ALTER TABLE interaction_approvals ADD COLUMN IF NOT EXISTS consumed_at timestamptz`,
|
|
593
|
+
// Claiming something nobody decided would be permission created out of nothing, so the state is
|
|
594
|
+
// excluded by the schema and not only by the UPDATE's WHERE clause.
|
|
595
|
+
`DO $$ BEGIN
|
|
596
|
+
ALTER TABLE interaction_approvals ADD CONSTRAINT interaction_approvals_consumed_after_decided
|
|
597
|
+
CHECK (consumed_at IS NULL OR decided_at IS NOT NULL);
|
|
598
|
+
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
|
|
599
|
+
// Serves findDecidedApproval: decided, not yet claimed -- the one row a resumed run looks for.
|
|
600
|
+
`CREATE INDEX IF NOT EXISTS interaction_approvals_resumable_run_idx
|
|
601
|
+
ON interaction_approvals (tenant_id, run_id)
|
|
602
|
+
WHERE decided_at IS NOT NULL AND consumed_at IS NULL`,
|
|
603
|
+
],
|
|
604
|
+
down: [
|
|
605
|
+
`DROP INDEX IF EXISTS interaction_approvals_resumable_run_idx`,
|
|
606
|
+
`ALTER TABLE interaction_approvals DROP CONSTRAINT IF EXISTS interaction_approvals_consumed_after_decided`,
|
|
607
|
+
`ALTER TABLE interaction_approvals DROP COLUMN IF EXISTS consumed_at`,
|
|
608
|
+
],
|
|
609
|
+
},
|
|
610
|
+
{
|
|
611
|
+
// Attachment metadata (#129). The bytes are not here, deliberately: `FileContentStore` is an
|
|
612
|
+
// object-storage port, and a relational adapter holding file bytes means base64 in a column -- the
|
|
613
|
+
// antipattern 0011 rejected when it declined to make `blobs` a pointer table.
|
|
614
|
+
id: "0013_files",
|
|
615
|
+
up: [
|
|
616
|
+
`CREATE TABLE IF NOT EXISTS files (
|
|
617
|
+
tenant_id text NOT NULL,
|
|
618
|
+
id text NOT NULL,
|
|
619
|
+
conversation_id text NOT NULL,
|
|
620
|
+
filename text NOT NULL,
|
|
621
|
+
media_type text NOT NULL,
|
|
622
|
+
byte_size bigint NOT NULL CHECK (byte_size >= 0),
|
|
623
|
+
-- Opaque, and unique per tenant: two rows pointing at one object would let deleting either one
|
|
624
|
+
-- take the other's bytes with it.
|
|
625
|
+
content_key text NOT NULL,
|
|
626
|
+
checksum text,
|
|
627
|
+
state text NOT NULL CHECK (state IN ('pending','stored','deleting','deleted')),
|
|
628
|
+
uploaded_by text NOT NULL,
|
|
629
|
+
created_at timestamptz NOT NULL,
|
|
630
|
+
deleted_at timestamptz,
|
|
631
|
+
PRIMARY KEY (tenant_id, id),
|
|
632
|
+
-- The conversation owns the file, so entitlement to one *is* entitlement to the other and there is
|
|
633
|
+
-- no second permission model. ON DELETE RESTRICT rather than CASCADE: a cascade would drop the
|
|
634
|
+
-- metadata and leave the bytes, which is precisely the orphan this design exists to avoid --
|
|
635
|
+
-- deletion goes through scheduleConversationDeletion so a sweep can find them.
|
|
636
|
+
FOREIGN KEY (tenant_id, conversation_id)
|
|
637
|
+
REFERENCES conversations (tenant_id, id) ON DELETE RESTRICT,
|
|
638
|
+
-- A row can only be 'deleted' with a timestamp saying when, so "deleted" is never a state without
|
|
639
|
+
-- a time.
|
|
640
|
+
CHECK (state <> 'deleted' OR deleted_at IS NOT NULL)
|
|
641
|
+
)`,
|
|
642
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS files_content_key_key ON files (tenant_id, content_key)`,
|
|
643
|
+
// Serves listByConversation's keyset page: live rows only, ordered the way the cursor reads.
|
|
644
|
+
`CREATE INDEX IF NOT EXISTS files_conversation_idx
|
|
645
|
+
ON files (tenant_id, conversation_id, created_at, id)
|
|
646
|
+
WHERE deleted_at IS NULL AND state <> 'deleted'`,
|
|
647
|
+
// Serves listByState, which reconciliation and the sweep both use. Partial on the two states anyone
|
|
648
|
+
// ever asks for, so it stays small next to a tenant's whole file history.
|
|
649
|
+
`CREATE INDEX IF NOT EXISTS files_pending_state_idx
|
|
650
|
+
ON files (tenant_id, state, created_at)
|
|
651
|
+
WHERE state IN ('pending','deleting')`,
|
|
652
|
+
],
|
|
653
|
+
down: [
|
|
654
|
+
`DROP INDEX IF EXISTS files_pending_state_idx`,
|
|
655
|
+
`DROP INDEX IF EXISTS files_conversation_idx`,
|
|
656
|
+
`DROP INDEX IF EXISTS files_content_key_key`,
|
|
657
|
+
`DROP TABLE IF EXISTS files`,
|
|
658
|
+
],
|
|
659
|
+
},
|
|
660
|
+
{
|
|
661
|
+
// #131. Extraction outcome on the file, as columns rather than a jsonb blob: `extraction_state` is
|
|
662
|
+
// queried by the reconciliation of stuck extractions, and a jsonb predicate there would be an index this
|
|
663
|
+
// schema does not have. The columns are all nullable because a file that nothing has tried to extract has
|
|
664
|
+
// no outcome -- distinct from an outcome of "pending", which means it is queued.
|
|
665
|
+
id: "0014_file_extraction",
|
|
666
|
+
up: [
|
|
667
|
+
`ALTER TABLE files ADD COLUMN IF NOT EXISTS extraction_state text`,
|
|
668
|
+
`ALTER TABLE files ADD COLUMN IF NOT EXISTS extraction_ref text`,
|
|
669
|
+
`ALTER TABLE files ADD COLUMN IF NOT EXISTS extraction_failure_reason text`,
|
|
670
|
+
`ALTER TABLE files ADD COLUMN IF NOT EXISTS extraction_failure_message text`,
|
|
671
|
+
`ALTER TABLE files ADD COLUMN IF NOT EXISTS extraction_pages integer`,
|
|
672
|
+
`ALTER TABLE files ADD COLUMN IF NOT EXISTS extraction_blocks integer`,
|
|
673
|
+
`ALTER TABLE files ADD COLUMN IF NOT EXISTS extraction_truncated boolean`,
|
|
674
|
+
`ALTER TABLE files ADD COLUMN IF NOT EXISTS extracted_at timestamptz`,
|
|
675
|
+
// #132. `real` rather than numeric: a confidence is a measurement, not money, and two decimal places
|
|
676
|
+
// of a float are all anyone acts on. Nullable because a PDF's text layer is read rather than
|
|
677
|
+
// recognised -- absent means "not probabilistic", not "unknown".
|
|
678
|
+
`ALTER TABLE files ADD COLUMN IF NOT EXISTS extraction_confidence real`,
|
|
679
|
+
`DO $$ BEGIN
|
|
680
|
+
ALTER TABLE files ADD CONSTRAINT files_extraction_confidence_ck
|
|
681
|
+
CHECK (extraction_confidence IS NULL OR (extraction_confidence >= 0 AND extraction_confidence <= 1));
|
|
682
|
+
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
|
|
683
|
+
// A failure without a reason is a failure nobody can act on, and a reason without a failure state is a
|
|
684
|
+
// row that contradicts itself. Both directions, in one constraint, because a check that only held one
|
|
685
|
+
// way would let the other through.
|
|
686
|
+
`DO $$ BEGIN
|
|
687
|
+
ALTER TABLE files ADD CONSTRAINT files_extraction_failure_ck
|
|
688
|
+
CHECK ((extraction_state = 'failed') = (extraction_failure_reason IS NOT NULL));
|
|
689
|
+
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
|
|
690
|
+
// Serves listByExtractionState. Partial on the two states reconciliation asks about -- a file that
|
|
691
|
+
// extracted successfully is never swept, so keeping it in the index would be most of the table for
|
|
692
|
+
// nothing. The COALESCE matches the query's, so a NULL state is found as 'pending' here too.
|
|
693
|
+
`CREATE INDEX IF NOT EXISTS files_extraction_state_idx
|
|
694
|
+
ON files (tenant_id, COALESCE(extraction_state, 'pending'), created_at, id)
|
|
695
|
+
WHERE extraction_state IS NULL OR extraction_state IN ('pending','running')`,
|
|
696
|
+
],
|
|
697
|
+
down: [
|
|
698
|
+
`DROP INDEX IF EXISTS files_extraction_state_idx`,
|
|
699
|
+
`ALTER TABLE files DROP CONSTRAINT IF EXISTS files_extraction_confidence_ck`,
|
|
700
|
+
`ALTER TABLE files DROP CONSTRAINT IF EXISTS files_extraction_failure_ck`,
|
|
701
|
+
`ALTER TABLE files DROP COLUMN IF EXISTS extraction_confidence`,
|
|
702
|
+
`ALTER TABLE files DROP COLUMN IF EXISTS extracted_at`,
|
|
703
|
+
`ALTER TABLE files DROP COLUMN IF EXISTS extraction_truncated`,
|
|
704
|
+
`ALTER TABLE files DROP COLUMN IF EXISTS extraction_blocks`,
|
|
705
|
+
`ALTER TABLE files DROP COLUMN IF EXISTS extraction_pages`,
|
|
706
|
+
`ALTER TABLE files DROP COLUMN IF EXISTS extraction_failure_message`,
|
|
707
|
+
`ALTER TABLE files DROP COLUMN IF EXISTS extraction_failure_reason`,
|
|
708
|
+
`ALTER TABLE files DROP COLUMN IF EXISTS extraction_ref`,
|
|
709
|
+
`ALTER TABLE files DROP COLUMN IF EXISTS extraction_state`,
|
|
710
|
+
],
|
|
711
|
+
},
|
|
712
|
+
{
|
|
713
|
+
// #133. Two tables, not one with a version column: an artifact has an identity that outlives any one
|
|
714
|
+
// version -- a name, an owning conversation, a shared link -- and folding them together would mean either
|
|
715
|
+
// duplicating that identity on every version or having no row to point a deleted link at.
|
|
716
|
+
id: "0015_artifacts",
|
|
717
|
+
up: [
|
|
718
|
+
`CREATE TABLE IF NOT EXISTS artifacts (
|
|
719
|
+
tenant_id text NOT NULL,
|
|
720
|
+
id text NOT NULL,
|
|
721
|
+
conversation_id text NOT NULL,
|
|
722
|
+
kind text NOT NULL,
|
|
723
|
+
name text NOT NULL,
|
|
724
|
+
-- Denormalised from artifact_versions on purpose. It is read on every resolve of "the current
|
|
725
|
+
-- version", and a MAX() subquery there would be a second source of truth that can disagree with the
|
|
726
|
+
-- rows it summarises under concurrency -- which is the exact race addVersion's compare-and-set exists
|
|
727
|
+
-- to settle.
|
|
728
|
+
latest_version integer NOT NULL,
|
|
729
|
+
created_at timestamptz NOT NULL,
|
|
730
|
+
updated_at timestamptz NOT NULL,
|
|
731
|
+
deleted_at timestamptz,
|
|
732
|
+
PRIMARY KEY (tenant_id, id),
|
|
733
|
+
-- RESTRICT, not CASCADE: dropping an artifact's row while its versions and blobs remain is the
|
|
734
|
+
-- orphan #129 refused for files, and for the same reason.
|
|
735
|
+
FOREIGN KEY (tenant_id, conversation_id) REFERENCES conversations (tenant_id, id) ON DELETE RESTRICT,
|
|
736
|
+
CHECK (latest_version >= 1)
|
|
737
|
+
)`,
|
|
738
|
+
`CREATE TABLE IF NOT EXISTS artifact_versions (
|
|
739
|
+
tenant_id text NOT NULL,
|
|
740
|
+
id text NOT NULL,
|
|
741
|
+
artifact_id text NOT NULL,
|
|
742
|
+
version integer NOT NULL,
|
|
743
|
+
-- A reference, never the content. An artifact is the thing a user exports, so it grows without limit,
|
|
744
|
+
-- and an unbounded value in a row is the antipattern 0011 rejected for blobs and 0013 for file bytes.
|
|
745
|
+
content_ref text NOT NULL,
|
|
746
|
+
byte_size bigint NOT NULL,
|
|
747
|
+
checksum text,
|
|
748
|
+
-- Provenance as jsonb because the inputs are arbitrary: what a tool was called with is that tool's
|
|
749
|
+
-- shape, and a column per input is a migration per tool.
|
|
750
|
+
provenance jsonb NOT NULL,
|
|
751
|
+
created_by text NOT NULL,
|
|
752
|
+
created_at timestamptz NOT NULL,
|
|
753
|
+
PRIMARY KEY (tenant_id, id),
|
|
754
|
+
FOREIGN KEY (tenant_id, artifact_id) REFERENCES artifacts (tenant_id, id) ON DELETE RESTRICT,
|
|
755
|
+
-- The constraint that makes AC-2 a property rather than a convention: two concurrent regenerations
|
|
756
|
+
-- cannot both be version 2, whatever the application layer believes.
|
|
757
|
+
UNIQUE (tenant_id, artifact_id, version),
|
|
758
|
+
CHECK (version >= 1),
|
|
759
|
+
CHECK (byte_size >= 0)
|
|
760
|
+
)`,
|
|
761
|
+
// Serves listByConversation's keyset page: live rows only, ordered the way the cursor reads.
|
|
762
|
+
`CREATE INDEX IF NOT EXISTS artifacts_conversation_idx
|
|
763
|
+
ON artifacts (tenant_id, conversation_id, created_at, id)
|
|
764
|
+
WHERE deleted_at IS NULL`,
|
|
765
|
+
// Serves listVersions and getVersion. The unique constraint above already indexes
|
|
766
|
+
// (tenant_id, artifact_id, version), so this exists only for the descending scan a "latest" lookup
|
|
767
|
+
// does -- and is dropped rather than kept if it ever shows as unused.
|
|
768
|
+
`CREATE INDEX IF NOT EXISTS artifact_versions_latest_idx
|
|
769
|
+
ON artifact_versions (tenant_id, artifact_id, version DESC)`,
|
|
770
|
+
],
|
|
771
|
+
down: [
|
|
772
|
+
`DROP INDEX IF EXISTS artifact_versions_latest_idx`,
|
|
773
|
+
`DROP INDEX IF EXISTS artifacts_conversation_idx`,
|
|
774
|
+
`DROP TABLE IF EXISTS artifact_versions`,
|
|
775
|
+
`DROP TABLE IF EXISTS artifacts`,
|
|
776
|
+
],
|
|
777
|
+
},
|
|
778
|
+
{
|
|
779
|
+
// #134. A rendered export of one artifact *version*, not a version of its own -- see the note on
|
|
780
|
+
// ArtifactExport for why a PDF is not an artifact version.
|
|
781
|
+
id: "0016_artifact_exports",
|
|
782
|
+
up: [
|
|
783
|
+
`CREATE TABLE IF NOT EXISTS artifact_exports (
|
|
784
|
+
tenant_id text NOT NULL,
|
|
785
|
+
id text NOT NULL,
|
|
786
|
+
artifact_id text NOT NULL,
|
|
787
|
+
version integer NOT NULL,
|
|
788
|
+
format text NOT NULL,
|
|
789
|
+
state text NOT NULL,
|
|
790
|
+
-- The rendered bytes, as a file. A FileId rather than a BlobRef because blobs hold JSON and a PDF is
|
|
791
|
+
-- bytes -- and going through the file ports means an export inherits 0013's entitlement check and
|
|
792
|
+
-- short-lived signed URLs instead of needing a second mediated-download path.
|
|
793
|
+
file_id text,
|
|
794
|
+
byte_size bigint,
|
|
795
|
+
checksum text,
|
|
796
|
+
failure_reason text,
|
|
797
|
+
failure_message text,
|
|
798
|
+
requested_by text NOT NULL,
|
|
799
|
+
created_at timestamptz NOT NULL,
|
|
800
|
+
rendered_at timestamptz,
|
|
801
|
+
PRIMARY KEY (tenant_id, id),
|
|
802
|
+
FOREIGN KEY (tenant_id, artifact_id) REFERENCES artifacts (tenant_id, id) ON DELETE RESTRICT,
|
|
803
|
+
-- The constraint that makes "re-downloaded without re-rendering" a property rather than a cache
|
|
804
|
+
-- someone remembers to check: one export per version per format, enforced where the application
|
|
805
|
+
-- cannot be wrong about it. Two concurrent requests for the same PDF cannot both claim it.
|
|
806
|
+
UNIQUE (tenant_id, artifact_id, version, format),
|
|
807
|
+
-- A rendered export without a file is a row promising a download that does not exist; a failed one
|
|
808
|
+
-- without a reason is a failure nobody can act on. Both directions, because a check that held one way
|
|
809
|
+
-- would let the other through.
|
|
810
|
+
CHECK ((state = 'rendered') = (file_id IS NOT NULL)),
|
|
811
|
+
CHECK ((state = 'failed') = (failure_reason IS NOT NULL)),
|
|
812
|
+
CHECK (version >= 1)
|
|
813
|
+
)`,
|
|
814
|
+
`CREATE INDEX IF NOT EXISTS artifact_exports_artifact_idx
|
|
815
|
+
ON artifact_exports (tenant_id, artifact_id, created_at, id)`,
|
|
816
|
+
],
|
|
817
|
+
down: [
|
|
818
|
+
`DROP INDEX IF EXISTS artifact_exports_artifact_idx`,
|
|
819
|
+
`DROP TABLE IF EXISTS artifact_exports`,
|
|
820
|
+
],
|
|
821
|
+
},
|
|
822
|
+
{
|
|
823
|
+
// #139. Rollups, keyed on (tenant, period, bucket_start). No extension needed, so this belongs in the main
|
|
824
|
+
// list rather than the optional vector one.
|
|
825
|
+
//
|
|
826
|
+
// Numbered 0018, not 0017: `VECTOR_MIGRATIONS` took 0017. The two lists are applied separately and each in
|
|
827
|
+
// its own order, so the gap is harmless — but the *numbers* must stay unique across both, because they are
|
|
828
|
+
// how a deployment records what it has run and a collision would make one migration look already applied.
|
|
829
|
+
id: "0018_usage_rollups",
|
|
830
|
+
up: [
|
|
831
|
+
`CREATE TABLE IF NOT EXISTS usage_rollups (
|
|
832
|
+
tenant_id text NOT NULL,
|
|
833
|
+
period text NOT NULL,
|
|
834
|
+
-- The instant the bucket opens, truncated to the period. Identified by its start rather than a range,
|
|
835
|
+
-- so two writers asking "which bucket does T belong to" agree by construction.
|
|
836
|
+
bucket_start timestamptz NOT NULL,
|
|
837
|
+
input_tokens bigint NOT NULL,
|
|
838
|
+
output_tokens bigint NOT NULL,
|
|
839
|
+
cached_input_tokens bigint NOT NULL,
|
|
840
|
+
reasoning_tokens bigint NOT NULL,
|
|
841
|
+
-- Integer minor units, per 0009's usage_records. Exact arithmetic: a float total of a million events is
|
|
842
|
+
-- wrong in the last cents, and cents are what an invoice is made of.
|
|
843
|
+
cost_minor_units bigint NOT NULL,
|
|
844
|
+
event_count bigint NOT NULL,
|
|
845
|
+
currency text NOT NULL,
|
|
846
|
+
computed_at timestamptz NOT NULL,
|
|
847
|
+
-- The primary key *is* the idempotency: a rebuild upserts onto it, so re-running a bucket cannot
|
|
848
|
+
-- create a second row and two workers racing one bucket write the same value.
|
|
849
|
+
PRIMARY KEY (tenant_id, period, bucket_start),
|
|
850
|
+
CHECK (period IN ('hour','day')),
|
|
851
|
+
CHECK (input_tokens >= 0 AND output_tokens >= 0 AND cost_minor_units >= 0 AND event_count >= 0)
|
|
852
|
+
)`,
|
|
853
|
+
// Serves the range read AC-1 is about: one tenant, one period, ordered the way a chart wants it. The
|
|
854
|
+
// primary key already covers this prefix, so this exists only for the descending scan a "latest bucket"
|
|
855
|
+
// lookup does.
|
|
856
|
+
`CREATE INDEX IF NOT EXISTS usage_rollups_range_idx
|
|
857
|
+
ON usage_rollups (tenant_id, period, bucket_start DESC)`,
|
|
858
|
+
// Serves the rebuild's scan of raw events by time. Without it every rebuild is a full table scan of the
|
|
859
|
+
// tenant's whole history, which is the cost the rollup exists to avoid paying on every read.
|
|
860
|
+
`CREATE INDEX IF NOT EXISTS usage_records_occurred_idx
|
|
861
|
+
ON usage_records (tenant_id, occurred_at)`,
|
|
862
|
+
],
|
|
863
|
+
down: [
|
|
864
|
+
`DROP INDEX IF EXISTS usage_records_occurred_idx`,
|
|
865
|
+
`DROP INDEX IF EXISTS usage_rollups_range_idx`,
|
|
866
|
+
`DROP TABLE IF EXISTS usage_rollups`,
|
|
867
|
+
],
|
|
868
|
+
},
|
|
869
|
+
{
|
|
870
|
+
// #139. When we *learned* about an event, as distinct from when it happened.
|
|
871
|
+
//
|
|
872
|
+
// Staleness has to be measured against this and not `occurred_at`. An event can be recorded late — a
|
|
873
|
+
// delayed provider report, a recovered run replaying its steps — with an `occurred_at` hours in the past.
|
|
874
|
+
// Judged by `occurred_at` its bucket looks already computed and the event is never rolled up: a silent
|
|
875
|
+
// undercount, in the direction that loses revenue. Found by writing the late-arrival test.
|
|
876
|
+
//
|
|
877
|
+
// `DEFAULT now()` so existing rows get a value and no backfill is needed; they are all older than any
|
|
878
|
+
// rollup, which correctly marks their buckets stale once and then settles.
|
|
879
|
+
id: "0019_usage_record_sequence",
|
|
880
|
+
up: [
|
|
881
|
+
// A monotonic sequence, not a timestamp.
|
|
882
|
+
//
|
|
883
|
+
// Staleness asks "has this bucket gained a record since it was last rolled up", and a *clock* cannot
|
|
884
|
+
// answer it: `now()` is constant within a transaction, `clock_timestamp()` ties under load, and either
|
|
885
|
+
// way equality has to be read as "stale" to stay safe — which makes a bucket that genuinely drained
|
|
886
|
+
// indistinguishable from one that did not. A sequence has no ties and no clock skew, and it is exactly
|
|
887
|
+
// the fact being compared: this rollup covers records up to N.
|
|
888
|
+
//
|
|
889
|
+
// It also closes a read anomaly the timestamp version could not: a row committed *during* a rebuild gets
|
|
890
|
+
// a higher sequence than the rollup records, so the bucket is correctly still stale — where its
|
|
891
|
+
// `recorded_at` could plausibly have been earlier than the rollup's stamp.
|
|
892
|
+
`ALTER TABLE usage_records ADD COLUMN IF NOT EXISTS record_seq bigserial`,
|
|
893
|
+
`CREATE INDEX IF NOT EXISTS usage_records_seq_idx ON usage_records (tenant_id, record_seq)`,
|
|
894
|
+
// The high-water mark a rollup covers. NULL for a bucket written before this migration, which correctly
|
|
895
|
+
// reads as "covers nothing" and rebuilds once.
|
|
896
|
+
`ALTER TABLE usage_rollups ADD COLUMN IF NOT EXISTS covers_seq bigint`,
|
|
897
|
+
],
|
|
898
|
+
down: [
|
|
899
|
+
`ALTER TABLE usage_rollups DROP COLUMN IF EXISTS covers_seq`,
|
|
900
|
+
`DROP INDEX IF EXISTS usage_records_seq_idx`,
|
|
901
|
+
`ALTER TABLE usage_records DROP COLUMN IF EXISTS record_seq`,
|
|
902
|
+
],
|
|
903
|
+
},
|
|
904
|
+
{
|
|
905
|
+
// #141. Evaluation runs and their per-case results.
|
|
906
|
+
id: "0020_evaluation_runs",
|
|
907
|
+
up: [
|
|
908
|
+
`CREATE TABLE IF NOT EXISTS evaluation_runs (
|
|
909
|
+
tenant_id text NOT NULL,
|
|
910
|
+
id text NOT NULL,
|
|
911
|
+
release text NOT NULL,
|
|
912
|
+
started_at timestamptz NOT NULL,
|
|
913
|
+
-- NULL while the run is in flight. A comparison must exclude those: an unfinished run's totals are
|
|
914
|
+
-- partial, and every case it has not reached yet would read as a regression.
|
|
915
|
+
finished_at timestamptz,
|
|
916
|
+
total integer NOT NULL,
|
|
917
|
+
passed integer NOT NULL,
|
|
918
|
+
-- The mean, stored rather than derived. A per-dimension breakdown of a thousand cases is read every
|
|
919
|
+
-- release, and once a case is retired from the dataset the historical mean is no longer recomputable.
|
|
920
|
+
mean_score double precision NOT NULL,
|
|
921
|
+
by_dimension jsonb NOT NULL,
|
|
922
|
+
cost_minor_units bigint NOT NULL,
|
|
923
|
+
-- Which graders produced these numbers. Without it, a comparison across a grader change is a comparison
|
|
924
|
+
-- of two instruments and nobody can tell.
|
|
925
|
+
grader_versions jsonb NOT NULL,
|
|
926
|
+
PRIMARY KEY (tenant_id, id),
|
|
927
|
+
CHECK (passed <= total),
|
|
928
|
+
CHECK (mean_score >= 0 AND mean_score <= 1)
|
|
929
|
+
)`,
|
|
930
|
+
`CREATE TABLE IF NOT EXISTS evaluation_case_results (
|
|
931
|
+
tenant_id text NOT NULL,
|
|
932
|
+
run_id text NOT NULL,
|
|
933
|
+
case_id text NOT NULL,
|
|
934
|
+
dimension text NOT NULL,
|
|
935
|
+
expect_kind text NOT NULL,
|
|
936
|
+
passed boolean NOT NULL,
|
|
937
|
+
score double precision NOT NULL,
|
|
938
|
+
reason text NOT NULL,
|
|
939
|
+
grader_id text NOT NULL,
|
|
940
|
+
grader_version text NOT NULL,
|
|
941
|
+
model_id text,
|
|
942
|
+
prompt_version text,
|
|
943
|
+
cost_minor_units bigint NOT NULL,
|
|
944
|
+
-- The key *is* the idempotency: a resumed run re-recording a case it already scored upserts onto this
|
|
945
|
+
-- rather than double-counting it in the aggregate.
|
|
946
|
+
PRIMARY KEY (tenant_id, run_id, case_id),
|
|
947
|
+
FOREIGN KEY (tenant_id, run_id) REFERENCES evaluation_runs (tenant_id, id) ON DELETE CASCADE,
|
|
948
|
+
CHECK (score >= 0 AND score <= 1)
|
|
949
|
+
)`,
|
|
950
|
+
// Serves latest(): the most recent completed run, optionally for one release.
|
|
951
|
+
`CREATE INDEX IF NOT EXISTS evaluation_runs_release_idx
|
|
952
|
+
ON evaluation_runs (tenant_id, release, finished_at DESC)
|
|
953
|
+
WHERE finished_at IS NOT NULL`,
|
|
954
|
+
// Serves the per-dimension read a comparison does.
|
|
955
|
+
`CREATE INDEX IF NOT EXISTS evaluation_case_results_dimension_idx
|
|
956
|
+
ON evaluation_case_results (tenant_id, run_id, dimension)`,
|
|
957
|
+
],
|
|
958
|
+
down: [
|
|
959
|
+
`DROP INDEX IF EXISTS evaluation_case_results_dimension_idx`,
|
|
960
|
+
`DROP INDEX IF EXISTS evaluation_runs_release_idx`,
|
|
961
|
+
`DROP TABLE IF EXISTS evaluation_case_results`,
|
|
962
|
+
`DROP TABLE IF EXISTS evaluation_runs`,
|
|
963
|
+
],
|
|
964
|
+
},
|
|
965
|
+
{
|
|
966
|
+
// #151 — the index the retention sweep needs.
|
|
967
|
+
//
|
|
968
|
+
// `run_events` is the fastest-growing table in the schema: one row per streamed part, so a single long
|
|
969
|
+
// assistant turn writes hundreds. The primary key is `(tenant_id, run_id, sequence)`, which cannot serve an
|
|
970
|
+
// age-based scan at all -- `created_at` is not a prefix of it, so a sweep without this index is a `Seq Scan`
|
|
971
|
+
// over the largest table there is.
|
|
972
|
+
id: "0021_run_events_retention",
|
|
973
|
+
up: [
|
|
974
|
+
// `(created_at, tenant_id, run_id)` rather than `(created_at)` alone.
|
|
975
|
+
//
|
|
976
|
+
// The leading column serves the range predicate, which is the whole point. The two trailing columns are
|
|
977
|
+
// the join key the safety predicate needs — the sweep has to look up each candidate's run to check it is
|
|
978
|
+
// terminal — so including them lets the candidate scan be **index-only**, with no heap fetch per row. On a
|
|
979
|
+
// table where the interesting case is "millions of rows, most of them prunable", the difference between an
|
|
980
|
+
// index-only scan and a heap fetch per candidate is the difference between a sweep that finishes and one
|
|
981
|
+
// that thrashes the buffer cache.
|
|
982
|
+
//
|
|
983
|
+
// Verified with EXPLAIN in `run-events-retention.test.ts` against a real server, not assumed. The
|
|
984
|
+
// #93/#94 pattern: an index added on faith is an index nobody knows is being used.
|
|
985
|
+
`CREATE INDEX IF NOT EXISTS run_events_created_at_idx
|
|
986
|
+
ON run_events (created_at, tenant_id, run_id)`,
|
|
987
|
+
// Serves the safety join from the other side: "is this run terminal?" by (tenant_id, id) is already the
|
|
988
|
+
// primary key of `runs`, so no index is needed there -- stated so the absence is a decision.
|
|
989
|
+
],
|
|
990
|
+
down: [`DROP INDEX IF EXISTS run_events_created_at_idx`],
|
|
991
|
+
},
|
|
992
|
+
{
|
|
993
|
+
id: "0022_run_principal",
|
|
994
|
+
up: [
|
|
995
|
+
// Who a run belongs to — #164.
|
|
996
|
+
//
|
|
997
|
+
// A run carried a tenant and no principal, so a durable worker had nothing to rebuild the caller's
|
|
998
|
+
// identity from. `DurableWorkerDeps.buildContext(run)` receives the run and must return an
|
|
999
|
+
// `ExecutionContext`, which requires a principal and roles: every host was therefore forced to invent
|
|
1000
|
+
// them, and the shipped example invented `principalId: "example-worker"` with `roleIds: ["editor"]`.
|
|
1001
|
+
//
|
|
1002
|
+
// Two consequences, both real. Every principal's memories, usage and audit entries were attributed to one
|
|
1003
|
+
// fabricated identity. And a `viewer` whose run was admitted at the API boundary executed in the worker
|
|
1004
|
+
// with editor rights, because the worker re-authorizes with the context it was handed.
|
|
1005
|
+
//
|
|
1006
|
+
// Nullable, because rows written before this migration have no answer and inventing one would be the same
|
|
1007
|
+
// mistake at the storage layer. A run with no principal is a run from before identity was recorded, and
|
|
1008
|
+
// `parseExecutionContext` refusing it is better than a silent substitution.
|
|
1009
|
+
`ALTER TABLE runs ADD COLUMN IF NOT EXISTS principal_id text`,
|
|
1010
|
+
// `text[]`, not jsonb: this is a list of role ids, and an array is the type Postgres has for that. jsonb
|
|
1011
|
+
// would allow `{"not": "a list"}` to be stored and only fail on read.
|
|
1012
|
+
`ALTER TABLE runs ADD COLUMN IF NOT EXISTS role_ids text[]`,
|
|
1013
|
+
],
|
|
1014
|
+
down: [
|
|
1015
|
+
`ALTER TABLE runs DROP COLUMN IF EXISTS role_ids`,
|
|
1016
|
+
`ALTER TABLE runs DROP COLUMN IF EXISTS principal_id`,
|
|
1017
|
+
],
|
|
1018
|
+
},
|
|
1019
|
+
{
|
|
1020
|
+
id: "0023_usage_per_principal",
|
|
1021
|
+
up: [
|
|
1022
|
+
// Who consumed it — #175.
|
|
1023
|
+
//
|
|
1024
|
+
// A usage record carried a tenant and no principal, so "what has this person spent" was unanswerable: the
|
|
1025
|
+
// data was never recorded. Nullable, because rows written before this exist and a record with an unknown
|
|
1026
|
+
// principal is a fact rather than something to invent.
|
|
1027
|
+
`ALTER TABLE usage_records ADD COLUMN IF NOT EXISTS principal_id text`,
|
|
1028
|
+
// Serves the per-principal breakdown and the per-principal rollup rebuild. `occurred_at` trails the
|
|
1029
|
+
// principal so a range scan for one person is a range scan rather than a filter over the tenant.
|
|
1030
|
+
`CREATE INDEX IF NOT EXISTS usage_records_principal_idx
|
|
1031
|
+
ON usage_records (tenant_id, principal_id, occurred_at)`,
|
|
1032
|
+
// Week and month buckets. The existing CHECK named only hour and day, so a weekly rollup would have been
|
|
1033
|
+
// rejected by the constraint rather than by any code — the least discoverable kind of failure.
|
|
1034
|
+
`ALTER TABLE usage_rollups DROP CONSTRAINT IF EXISTS usage_rollups_period_check`,
|
|
1035
|
+
`ALTER TABLE usage_rollups ADD CONSTRAINT usage_rollups_period_check
|
|
1036
|
+
CHECK (period IN ('hour','day','week','month'))`,
|
|
1037
|
+
/**
|
|
1038
|
+
* Per-principal rollups, as a nullable dimension on the existing table.
|
|
1039
|
+
*
|
|
1040
|
+
* `principal_id IS NULL` is the **tenant total**; a non-null row is one person's. That is the standard
|
|
1041
|
+
* grouping-set shape, and it is what lets a per-principal quota read one row on the admission path instead
|
|
1042
|
+
* of scanning the ledger — which is the whole reason rollups exist.
|
|
1043
|
+
*
|
|
1044
|
+
* Nullable rather than a separate table because the two are the same measurement at two grains: a second
|
|
1045
|
+
* table would mean two rebuild paths, and the day they disagree is the day an invoice is wrong.
|
|
1046
|
+
*
|
|
1047
|
+
* The primary key has to change to include it, and a NULL is not comparable in a primary key — so the
|
|
1048
|
+
* uniqueness is expressed as two partial unique indexes instead, one for the tenant row and one per
|
|
1049
|
+
* principal. `COALESCE` in a single index would work too and would hide which row is which.
|
|
1050
|
+
*/
|
|
1051
|
+
`ALTER TABLE usage_rollups ADD COLUMN IF NOT EXISTS principal_id text`,
|
|
1052
|
+
`ALTER TABLE usage_rollups DROP CONSTRAINT IF EXISTS usage_rollups_pkey`,
|
|
1053
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS usage_rollups_tenant_bucket_idx
|
|
1054
|
+
ON usage_rollups (tenant_id, period, bucket_start)
|
|
1055
|
+
WHERE principal_id IS NULL`,
|
|
1056
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS usage_rollups_principal_bucket_idx
|
|
1057
|
+
ON usage_rollups (tenant_id, principal_id, period, bucket_start)
|
|
1058
|
+
WHERE principal_id IS NOT NULL`,
|
|
1059
|
+
/**
|
|
1060
|
+
* Admin-configured limits — #175.
|
|
1061
|
+
*
|
|
1062
|
+
* `principal_id IS NULL` is the tenant-wide default and a non-null row overrides it for one person, which
|
|
1063
|
+
* is the same nullable-dimension shape as the rollups above. One table rather than two because "the
|
|
1064
|
+
* default" and "an override" are the same kind of fact, and resolution is then one query ordered by
|
|
1065
|
+
* specificity rather than two queries and a merge.
|
|
1066
|
+
*
|
|
1067
|
+
* Every limit column is nullable, and NULL means **unbounded** rather than zero. That direction is
|
|
1068
|
+
* deliberate and matches `QuotaLimits`: a misconfigured quota that blocks everything is an outage, and one
|
|
1069
|
+
* that blocks nothing is a bill — and the bill is visible in the rollups, whereas the outage is only
|
|
1070
|
+
* visible to the customer it is happening to.
|
|
1071
|
+
*/
|
|
1072
|
+
`CREATE TABLE IF NOT EXISTS usage_limits (
|
|
1073
|
+
tenant_id text NOT NULL,
|
|
1074
|
+
principal_id text,
|
|
1075
|
+
period text NOT NULL,
|
|
1076
|
+
cost_minor_units bigint,
|
|
1077
|
+
input_tokens bigint,
|
|
1078
|
+
output_tokens bigint,
|
|
1079
|
+
warn_at real,
|
|
1080
|
+
updated_at timestamptz NOT NULL,
|
|
1081
|
+
-- Who changed it. A spend limit is the kind of setting whose history someone eventually has to explain.
|
|
1082
|
+
updated_by text,
|
|
1083
|
+
CHECK (period IN ('hour','day','week','month')),
|
|
1084
|
+
-- A negative limit would refuse every run while reading like a configuration.
|
|
1085
|
+
CHECK (cost_minor_units IS NULL OR cost_minor_units >= 0),
|
|
1086
|
+
CHECK (input_tokens IS NULL OR input_tokens >= 0),
|
|
1087
|
+
CHECK (output_tokens IS NULL OR output_tokens >= 0),
|
|
1088
|
+
-- A warn threshold outside (0, 1] is a fraction that can never fire, or one that fires always.
|
|
1089
|
+
CHECK (warn_at IS NULL OR (warn_at > 0 AND warn_at <= 1))
|
|
1090
|
+
)`,
|
|
1091
|
+
/**
|
|
1092
|
+
* Guarded on the column still existing, because a later migration takes it away.
|
|
1093
|
+
*
|
|
1094
|
+
* `migrate` has no ledger and re-runs every statement, so a statement here has to be idempotent *and*
|
|
1095
|
+
* tolerant of what later migrations do. #181 renamed `period` to `window_key` and #182 replaced both of
|
|
1096
|
+
* these partial indexes with one expression index — so on a second pass `IF NOT EXISTS` did not skip
|
|
1097
|
+
* these (the indexes really were gone) and they failed on `column "period" does not exist`.
|
|
1098
|
+
*
|
|
1099
|
+
* That is the whole failure mode: an early migration re-creating something a later one deliberately
|
|
1100
|
+
* removed. Left unguarded it would also *restore* indexes that are now wrong — they enforce uniqueness
|
|
1101
|
+
* without the model dimension, so a model-scoped limit would collide with the unscoped one for the same
|
|
1102
|
+
* principal and window.
|
|
1103
|
+
*/
|
|
1104
|
+
`DO $$ BEGIN
|
|
1105
|
+
IF EXISTS (SELECT 1 FROM information_schema.columns
|
|
1106
|
+
WHERE table_schema = current_schema() AND table_name = 'usage_limits'
|
|
1107
|
+
AND column_name = 'period') THEN
|
|
1108
|
+
CREATE UNIQUE INDEX IF NOT EXISTS usage_limits_tenant_idx
|
|
1109
|
+
ON usage_limits (tenant_id, period) WHERE principal_id IS NULL;
|
|
1110
|
+
CREATE UNIQUE INDEX IF NOT EXISTS usage_limits_principal_idx
|
|
1111
|
+
ON usage_limits (tenant_id, principal_id, period) WHERE principal_id IS NOT NULL;
|
|
1112
|
+
END IF;
|
|
1113
|
+
END $$`,
|
|
1114
|
+
],
|
|
1115
|
+
down: [
|
|
1116
|
+
`DROP TABLE IF EXISTS usage_limits`,
|
|
1117
|
+
`DROP INDEX IF EXISTS usage_rollups_principal_bucket_idx`,
|
|
1118
|
+
`DROP INDEX IF EXISTS usage_rollups_tenant_bucket_idx`,
|
|
1119
|
+
`ALTER TABLE usage_rollups DROP COLUMN IF EXISTS principal_id`,
|
|
1120
|
+
`DROP INDEX IF EXISTS usage_records_principal_idx`,
|
|
1121
|
+
`ALTER TABLE usage_records DROP COLUMN IF EXISTS principal_id`,
|
|
1122
|
+
],
|
|
1123
|
+
},
|
|
1124
|
+
{
|
|
1125
|
+
/**
|
|
1126
|
+
* A limit's span becomes a **window key** — #181.
|
|
1127
|
+
*
|
|
1128
|
+
* `period` held one of four calendar periods and was already the key both unique indexes are built on. A
|
|
1129
|
+
* rolling window ("no more than X in any 5 hours") is another span, so it belongs in the same slot rather
|
|
1130
|
+
* than in a parallel nullable column — which would have made the unique indexes NULL-keyed, and Postgres
|
|
1131
|
+
* treats NULLs as distinct, so the upsert `put` relies on would have stopped deduplicating.
|
|
1132
|
+
*
|
|
1133
|
+
* Renamed rather than reused under the old name: `period = 'rolling:300'` is not a period, and a column whose
|
|
1134
|
+
* name is a lie is how the next person writes a wrong query. The indexes follow the rename automatically.
|
|
1135
|
+
*
|
|
1136
|
+
* The CHECK is widened, not dropped. `rolling:<minutes>` with 1 to 6 digits — an upper bound so a typo
|
|
1137
|
+
* cannot store a window of ten million minutes, and no leading zero so one value has one spelling and the
|
|
1138
|
+
* key stays unique in the way the index assumes.
|
|
1139
|
+
*/
|
|
1140
|
+
id: "0024_usage_limit_window",
|
|
1141
|
+
up: [
|
|
1142
|
+
/**
|
|
1143
|
+
* Guarded, because `migrate` re-runs **every** statement every time.
|
|
1144
|
+
*
|
|
1145
|
+
* There is no applied-migrations ledger: `migrate` walks the whole list, which works because every other
|
|
1146
|
+
* statement in this file is `IF NOT EXISTS`. `RENAME COLUMN` and `ADD CONSTRAINT` have no such form, so the
|
|
1147
|
+
* first version of this migration succeeded once and then failed on every subsequent run with
|
|
1148
|
+
* `column "period" does not exist` — which is how the example's `npm run migrate` broke.
|
|
1149
|
+
*
|
|
1150
|
+
* `current_schema()` rather than a literal: this schema is created under whatever `search_path` the caller
|
|
1151
|
+
* set, and the example deliberately runs in its own.
|
|
1152
|
+
*/
|
|
1153
|
+
`DO $$ BEGIN
|
|
1154
|
+
IF EXISTS (SELECT 1 FROM information_schema.columns
|
|
1155
|
+
WHERE table_schema = current_schema() AND table_name = 'usage_limits'
|
|
1156
|
+
AND column_name = 'period') THEN
|
|
1157
|
+
ALTER TABLE usage_limits RENAME COLUMN period TO window_key;
|
|
1158
|
+
END IF;
|
|
1159
|
+
END $$`,
|
|
1160
|
+
// Postgres named the anonymous single-column CHECK after the column, and a rename does not rename the
|
|
1161
|
+
// constraint — so it is still `usage_limits_period_check` here.
|
|
1162
|
+
`ALTER TABLE usage_limits DROP CONSTRAINT IF EXISTS usage_limits_period_check`,
|
|
1163
|
+
`DO $$ BEGIN
|
|
1164
|
+
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'usage_limits_window_key_check') THEN
|
|
1165
|
+
ALTER TABLE usage_limits ADD CONSTRAINT usage_limits_window_key_check
|
|
1166
|
+
CHECK (window_key IN ('hour','day','week','month') OR window_key ~ '^rolling:[1-9][0-9]{0,5}$');
|
|
1167
|
+
END IF;
|
|
1168
|
+
END $$`,
|
|
1169
|
+
],
|
|
1170
|
+
down: [
|
|
1171
|
+
// Rolling rows cannot survive a rollback: the old CHECK has no room for them. Deleted rather than left to
|
|
1172
|
+
// fail the constraint, because a migration that cannot go back is a migration nobody dares run forward.
|
|
1173
|
+
`DELETE FROM usage_limits WHERE window_key LIKE 'rolling:%'`,
|
|
1174
|
+
`ALTER TABLE usage_limits DROP CONSTRAINT IF EXISTS usage_limits_window_key_check`,
|
|
1175
|
+
`DO $$ BEGIN
|
|
1176
|
+
IF EXISTS (SELECT 1 FROM information_schema.columns
|
|
1177
|
+
WHERE table_schema = current_schema() AND table_name = 'usage_limits'
|
|
1178
|
+
AND column_name = 'window_key') THEN
|
|
1179
|
+
ALTER TABLE usage_limits RENAME COLUMN window_key TO period;
|
|
1180
|
+
END IF;
|
|
1181
|
+
END $$`,
|
|
1182
|
+
`DO $$ BEGIN
|
|
1183
|
+
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'usage_limits_period_check') THEN
|
|
1184
|
+
ALTER TABLE usage_limits ADD CONSTRAINT usage_limits_period_check
|
|
1185
|
+
CHECK (period IN ('hour','day','week','month'));
|
|
1186
|
+
END IF;
|
|
1187
|
+
END $$`,
|
|
1188
|
+
],
|
|
1189
|
+
},
|
|
1190
|
+
{
|
|
1191
|
+
/**
|
|
1192
|
+
* A limit can be scoped to a model — #182.
|
|
1193
|
+
*
|
|
1194
|
+
* An expensive model and a cheap one sharing one allowance is the opposite of what a per-model limit is for.
|
|
1195
|
+
* `usage_records.model_id` is required on every row, so the counting side needs nothing new; this is the
|
|
1196
|
+
* *limit* learning the dimension.
|
|
1197
|
+
*
|
|
1198
|
+
* **The two partial unique indexes collapse into one expression index.** They existed because a NULL cannot
|
|
1199
|
+
* participate in a normal unique constraint, so `principal_id IS NULL` needed its own — and `put` branched on
|
|
1200
|
+
* grain to name the right `ON CONFLICT` target. A second nullable dimension would have made that four
|
|
1201
|
+
* indexes and a 2x2 branch, which is four chances to name the wrong one; naming the wrong one is how #177's
|
|
1202
|
+
* `error: dup` reached a user. `COALESCE(col, '')` normalises both dimensions into one key instead, so there
|
|
1203
|
+
* is one index, one conflict target and no branch.
|
|
1204
|
+
*
|
|
1205
|
+
* The empty string is safe as the "absent" marker here because neither a principal id nor a model id can be
|
|
1206
|
+
* empty: both come from `asId`, and an empty one would already be a bug upstream. Existing rows have
|
|
1207
|
+
* `model_id NULL`, so their keys are unchanged by this.
|
|
1208
|
+
*/
|
|
1209
|
+
id: "0025_usage_limit_model",
|
|
1210
|
+
up: [
|
|
1211
|
+
`ALTER TABLE usage_limits ADD COLUMN IF NOT EXISTS model_id text`,
|
|
1212
|
+
// A model id that is present but empty would collide with the "no model" marker, so it is refused at the
|
|
1213
|
+
// boundary rather than silently becoming a tenant-wide limit.
|
|
1214
|
+
// Guarded for the same reason as 0024: `ADD CONSTRAINT` has no `IF NOT EXISTS`, and every statement here
|
|
1215
|
+
// runs on every `migrate`.
|
|
1216
|
+
`DO $$ BEGIN
|
|
1217
|
+
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'usage_limits_model_id_check') THEN
|
|
1218
|
+
ALTER TABLE usage_limits ADD CONSTRAINT usage_limits_model_id_check
|
|
1219
|
+
CHECK (model_id IS NULL OR length(model_id) > 0);
|
|
1220
|
+
END IF;
|
|
1221
|
+
END $$`,
|
|
1222
|
+
`DROP INDEX IF EXISTS usage_limits_tenant_idx`,
|
|
1223
|
+
`DROP INDEX IF EXISTS usage_limits_principal_idx`,
|
|
1224
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS usage_limits_scope_idx ON usage_limits
|
|
1225
|
+
(tenant_id, COALESCE(principal_id, ''), COALESCE(model_id, ''), window_key)`,
|
|
1226
|
+
],
|
|
1227
|
+
down: [
|
|
1228
|
+
// Model-scoped rows cannot survive the rollback: the old indexes cannot tell them apart from the
|
|
1229
|
+
// unscoped row for the same principal and window, so they would collide on re-creation.
|
|
1230
|
+
`DELETE FROM usage_limits WHERE model_id IS NOT NULL`,
|
|
1231
|
+
`DROP INDEX IF EXISTS usage_limits_scope_idx`,
|
|
1232
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS usage_limits_tenant_idx
|
|
1233
|
+
ON usage_limits (tenant_id, window_key) WHERE principal_id IS NULL`,
|
|
1234
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS usage_limits_principal_idx
|
|
1235
|
+
ON usage_limits (tenant_id, principal_id, window_key) WHERE principal_id IS NOT NULL`,
|
|
1236
|
+
`ALTER TABLE usage_limits DROP CONSTRAINT IF EXISTS usage_limits_model_id_check`,
|
|
1237
|
+
`ALTER TABLE usage_limits DROP COLUMN IF EXISTS model_id`,
|
|
1238
|
+
],
|
|
1239
|
+
},
|
|
1240
|
+
{
|
|
1241
|
+
/**
|
|
1242
|
+
* A run need not belong to a conversation — #198.
|
|
1243
|
+
*
|
|
1244
|
+
* `conversation_id NOT NULL` forced a triggered automation to invent a conversation id to exist. That is
|
|
1245
|
+
* #164's shape: `runs` carried no principal, so hosts fabricated one, and every per-person figure silently
|
|
1246
|
+
* became a machine's. A fabricated conversation id would do the same to every conversation-scoped query,
|
|
1247
|
+
* and it would look exactly like data.
|
|
1248
|
+
*
|
|
1249
|
+
* Dropping NOT NULL only. The column stays, the index stays, and every existing row is unaffected — which
|
|
1250
|
+
* is why this is safe to run against a live database.
|
|
1251
|
+
*/
|
|
1252
|
+
id: "0026_run_without_conversation",
|
|
1253
|
+
up: [`ALTER TABLE runs ALTER COLUMN conversation_id DROP NOT NULL`],
|
|
1254
|
+
down: [
|
|
1255
|
+
// Rows with no conversation cannot survive the constraint coming back, and there is no id to give them
|
|
1256
|
+
// that would not be a lie. Deleted, deliberately, rather than blocking the rollback.
|
|
1257
|
+
`DELETE FROM runs WHERE conversation_id IS NULL`,
|
|
1258
|
+
`ALTER TABLE runs ALTER COLUMN conversation_id SET NOT NULL`,
|
|
1259
|
+
],
|
|
1260
|
+
},
|
|
1261
|
+
{
|
|
1262
|
+
/**
|
|
1263
|
+
* Non-text input, recorded — REQ-036 (#185), AC-4.
|
|
1264
|
+
*
|
|
1265
|
+
* The cost is already right without these columns, because `computeModelCostMinorUnits` charges by whichever
|
|
1266
|
+
* convention the pricing record declares. What they add is *auditability*: a multimodal turn and a text turn
|
|
1267
|
+
* that happened to cost the same are otherwise indistinguishable in the ledger, so "why was this run
|
|
1268
|
+
* expensive" has no answer and a pricing mistake is invisible after the fact.
|
|
1269
|
+
*
|
|
1270
|
+
* Nullable, not `DEFAULT 0`. A zero would mean "no images" and an absent value means "this row was written
|
|
1271
|
+
* before anyone counted" — collapsing them would silently backdate a claim about rows nobody measured.
|
|
1272
|
+
*/
|
|
1273
|
+
id: "0027_usage_non_text_input",
|
|
1274
|
+
up: [
|
|
1275
|
+
`ALTER TABLE usage_records ADD COLUMN IF NOT EXISTS image_count integer`,
|
|
1276
|
+
`ALTER TABLE usage_records ADD COLUMN IF NOT EXISTS audio_seconds integer`,
|
|
1277
|
+
// Guarded like every other constraint here: `migrate` re-runs the whole list against a schema provisioned
|
|
1278
|
+
// before the ledger existed, so an unguarded ADD CONSTRAINT fails on the second pass.
|
|
1279
|
+
`DO $$
|
|
1280
|
+
BEGIN
|
|
1281
|
+
IF NOT EXISTS (
|
|
1282
|
+
SELECT 1 FROM pg_constraint WHERE conname = 'usage_records_non_text_non_negative'
|
|
1283
|
+
) THEN
|
|
1284
|
+
ALTER TABLE usage_records ADD CONSTRAINT usage_records_non_text_non_negative CHECK (
|
|
1285
|
+
(image_count IS NULL OR image_count >= 0) AND (audio_seconds IS NULL OR audio_seconds >= 0)
|
|
1286
|
+
);
|
|
1287
|
+
END IF;
|
|
1288
|
+
END $$`,
|
|
1289
|
+
],
|
|
1290
|
+
down: [
|
|
1291
|
+
`ALTER TABLE usage_records DROP CONSTRAINT IF EXISTS usage_records_non_text_non_negative`,
|
|
1292
|
+
`ALTER TABLE usage_records DROP COLUMN IF EXISTS audio_seconds`,
|
|
1293
|
+
`ALTER TABLE usage_records DROP COLUMN IF EXISTS image_count`,
|
|
1294
|
+
],
|
|
1295
|
+
},
|
|
1296
|
+
{
|
|
1297
|
+
/**
|
|
1298
|
+
* File bytes, for a deployment with no object storage — REQ-036 (#185).
|
|
1299
|
+
*
|
|
1300
|
+
* `bytea` and not a large object: `lo_*` needs its own transaction discipline and a separate vacuum story,
|
|
1301
|
+
* and the sizes this store is for are the ones a row holds comfortably. The `byte_size` check is what makes
|
|
1302
|
+
* a row that disagrees with its own bytes impossible rather than merely unlikely.
|
|
1303
|
+
*/
|
|
1304
|
+
id: "0028_file_objects",
|
|
1305
|
+
up: [
|
|
1306
|
+
`CREATE TABLE IF NOT EXISTS file_objects (
|
|
1307
|
+
tenant_id text NOT NULL,
|
|
1308
|
+
content_key text NOT NULL,
|
|
1309
|
+
media_type text NOT NULL,
|
|
1310
|
+
byte_size integer NOT NULL,
|
|
1311
|
+
checksum text NOT NULL,
|
|
1312
|
+
bytes bytea NOT NULL,
|
|
1313
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
1314
|
+
PRIMARY KEY (tenant_id, content_key),
|
|
1315
|
+
CONSTRAINT file_objects_size_matches CHECK (byte_size = octet_length(bytes)),
|
|
1316
|
+
CONSTRAINT file_objects_size_non_negative CHECK (byte_size >= 0)
|
|
1317
|
+
)`,
|
|
1318
|
+
// Reconciliation's second direction — bytes with no metadata — scans by tenant and prefix.
|
|
1319
|
+
`CREATE INDEX IF NOT EXISTS file_objects_tenant_key_idx ON file_objects (tenant_id, content_key)`,
|
|
1320
|
+
],
|
|
1321
|
+
down: [`DROP TABLE IF EXISTS file_objects`],
|
|
1322
|
+
},
|
|
1323
|
+
{
|
|
1324
|
+
/**
|
|
1325
|
+
* Flows and teams — REQ-038 (#187), REQ-037 (#186).
|
|
1326
|
+
*
|
|
1327
|
+
* Two constraints carry the guarantees rather than leaving them to the adapter:
|
|
1328
|
+
*
|
|
1329
|
+
* - `(tenant_id, flow_id, version)` is the primary key on definitions, and the insert has no `ON CONFLICT`.
|
|
1330
|
+
* A version cannot be overwritten, which is what makes an execution's version pin meaningful — a definition
|
|
1331
|
+
* that could change under a running execution would change an automation's shape halfway through.
|
|
1332
|
+
* - `steps >= 0`, and the save's `WHERE steps <= $n` predicate: an execution cannot be moved backwards by a
|
|
1333
|
+
* stale write from a slower worker. A flow that goes backwards re-performs external writes.
|
|
1334
|
+
*
|
|
1335
|
+
* A team is stored as a definition with `kind = 'team'`, because a team compiles to a flow and storing them
|
|
1336
|
+
* apart would mean two tables whose rows mean the same thing.
|
|
1337
|
+
*/
|
|
1338
|
+
id: "0029_flows",
|
|
1339
|
+
up: [
|
|
1340
|
+
`CREATE TABLE IF NOT EXISTS flow_definitions (
|
|
1341
|
+
tenant_id text NOT NULL,
|
|
1342
|
+
flow_id text NOT NULL,
|
|
1343
|
+
version integer NOT NULL,
|
|
1344
|
+
name text NOT NULL,
|
|
1345
|
+
kind text NOT NULL,
|
|
1346
|
+
definition jsonb NOT NULL,
|
|
1347
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
1348
|
+
created_by text,
|
|
1349
|
+
PRIMARY KEY (tenant_id, flow_id, version),
|
|
1350
|
+
CONSTRAINT flow_definitions_kind CHECK (kind IN ('flow', 'team')),
|
|
1351
|
+
CONSTRAINT flow_definitions_version_positive CHECK (version >= 1)
|
|
1352
|
+
)`,
|
|
1353
|
+
`CREATE TABLE IF NOT EXISTS flow_executions (
|
|
1354
|
+
tenant_id text NOT NULL,
|
|
1355
|
+
id text NOT NULL,
|
|
1356
|
+
flow_id text NOT NULL,
|
|
1357
|
+
flow_version integer NOT NULL,
|
|
1358
|
+
run_id text NOT NULL,
|
|
1359
|
+
status text NOT NULL,
|
|
1360
|
+
current_step text,
|
|
1361
|
+
steps integer NOT NULL DEFAULT 0,
|
|
1362
|
+
execution jsonb NOT NULL,
|
|
1363
|
+
waiting_signal text,
|
|
1364
|
+
started_at timestamptz NOT NULL DEFAULT now(),
|
|
1365
|
+
finished_at timestamptz,
|
|
1366
|
+
PRIMARY KEY (tenant_id, id),
|
|
1367
|
+
CONSTRAINT flow_executions_steps_non_negative CHECK (steps >= 0),
|
|
1368
|
+
CONSTRAINT flow_executions_status CHECK (status IN ('running', 'waiting', 'completed', 'failed', 'cancelled'))
|
|
1369
|
+
)`,
|
|
1370
|
+
// A delivered signal has to find what was waiting for it, and the inspector reads a flow's history.
|
|
1371
|
+
`CREATE INDEX IF NOT EXISTS flow_executions_signal_idx
|
|
1372
|
+
ON flow_executions (tenant_id, waiting_signal) WHERE status = 'waiting'`,
|
|
1373
|
+
`CREATE INDEX IF NOT EXISTS flow_executions_flow_idx ON flow_executions (tenant_id, flow_id, started_at DESC)`,
|
|
1374
|
+
],
|
|
1375
|
+
down: [`DROP TABLE IF EXISTS flow_executions`, `DROP TABLE IF EXISTS flow_definitions`],
|
|
1376
|
+
},
|
|
1377
|
+
{
|
|
1378
|
+
/**
|
|
1379
|
+
* A flow parked on a child run — #202.
|
|
1380
|
+
*
|
|
1381
|
+
* Its own column and its own partial index rather than a lookup inside the `execution` JSON: a settled run
|
|
1382
|
+
* has to find its parent, and that is a hot path on every run completion in a deployment that uses flows.
|
|
1383
|
+
* Scanning jsonb for it would make finishing a run slower for everyone, including deployments with no flows.
|
|
1384
|
+
*
|
|
1385
|
+
* The index is partial on `status = 'waiting'` for the same reason the query is: a stale id on a running
|
|
1386
|
+
* execution is not a parent waiting for anything.
|
|
1387
|
+
*/
|
|
1388
|
+
id: "0030_flow_waiting_run",
|
|
1389
|
+
up: [
|
|
1390
|
+
`ALTER TABLE flow_executions ADD COLUMN IF NOT EXISTS waiting_run_id text`,
|
|
1391
|
+
`CREATE INDEX IF NOT EXISTS flow_executions_waiting_run_idx
|
|
1392
|
+
ON flow_executions (tenant_id, waiting_run_id) WHERE status = 'waiting'`,
|
|
1393
|
+
],
|
|
1394
|
+
down: [
|
|
1395
|
+
`DROP INDEX IF EXISTS flow_executions_waiting_run_idx`,
|
|
1396
|
+
`ALTER TABLE flow_executions DROP COLUMN IF EXISTS waiting_run_id`,
|
|
1397
|
+
],
|
|
1398
|
+
},
|
|
1399
|
+
{
|
|
1400
|
+
/**
|
|
1401
|
+
* A run's own input and ceilings — #202.
|
|
1402
|
+
*
|
|
1403
|
+
* Both exist because a **conversation-less** run had nowhere to put them. #198 made a run able to exist
|
|
1404
|
+
* without a conversation — which a webhook, a schedule and a flow's agent step all are — but the only place a
|
|
1405
|
+
* request could live was a `Message`, and a message requires a conversation. So the run shape said "no
|
|
1406
|
+
* conversation needed" while the storage said its input still needed one.
|
|
1407
|
+
*
|
|
1408
|
+
* `limits` overrides the agent manifest's for this run only. A flow derives it from what the flow has left,
|
|
1409
|
+
* re-derived per step; without it a child takes the manifest's limits, which are a property of the *agent* and
|
|
1410
|
+
* therefore identical for every member of every team — so a team's budget would bound nothing.
|
|
1411
|
+
*/
|
|
1412
|
+
id: "0031_run_input_and_limits",
|
|
1413
|
+
up: [
|
|
1414
|
+
`ALTER TABLE runs ADD COLUMN IF NOT EXISTS input jsonb`,
|
|
1415
|
+
`ALTER TABLE runs ADD COLUMN IF NOT EXISTS limits jsonb`,
|
|
1416
|
+
],
|
|
1417
|
+
down: [
|
|
1418
|
+
`ALTER TABLE runs DROP COLUMN IF EXISTS limits`,
|
|
1419
|
+
`ALTER TABLE runs DROP COLUMN IF EXISTS input`,
|
|
1420
|
+
],
|
|
1421
|
+
},
|
|
1422
|
+
];
|
|
1423
|
+
/**
|
|
1424
|
+
* pgvector migrations, applied separately (#135).
|
|
1425
|
+
*
|
|
1426
|
+
* **Not in `MIGRATIONS`,** and that is deliberate. `CREATE EXTENSION vector` fails on a Postgres without
|
|
1427
|
+
* pgvector installed — including PGlite, which the conformance suite runs against by default — so folding
|
|
1428
|
+
* these into the main list would make every test require an extension most deployments install explicitly
|
|
1429
|
+
* anyway. `migrateVector` is called by a deployment that has it, and `hasVectorExtension` tells a caller
|
|
1430
|
+
* whether it does rather than making them guess from an error.
|
|
1431
|
+
*
|
|
1432
|
+
* **Dimensions are fixed at 1536.** pgvector's `vector(N)` needs a literal, and an index cannot span
|
|
1433
|
+
* dimensions, so a deployment changing embedding model *sizes* re-migrates rather than re-indexing —
|
|
1434
|
+
* which is why `EmbeddingModelRef` carries `dimensions` and `listStaleSources` compares it.
|
|
1435
|
+
*/
|
|
1436
|
+
export const VECTOR_DIMENSIONS = EMBEDDING_DIMENSIONS;
|
|
1437
|
+
/**
|
|
1438
|
+
* The index: HNSW, not IVFFlat.
|
|
1439
|
+
*
|
|
1440
|
+
* The benchmark the SPEC asks for, stated rather than implied. HNSW builds slower and uses more memory;
|
|
1441
|
+
* IVFFlat needs a representative sample at build time and degrades badly when the data grows past what it was
|
|
1442
|
+
* trained on. For a multi-tenant knowledge base where every tenant's corpus grows continuously and unevenly,
|
|
1443
|
+
* a list-based index has to be periodically rebuilt per tenant or recall silently falls — and "silently" is
|
|
1444
|
+
* the disqualifying part. HNSW's recall is stable as rows are added, which is worth its build cost here.
|
|
1445
|
+
*
|
|
1446
|
+
* `m = 16, ef_construction = 64` are pgvector's defaults and the operating point its own benchmarks report as
|
|
1447
|
+
* ~0.98 recall@10 on 1536-dimension embeddings. That figure is the recorded target; the measured figure for
|
|
1448
|
+
* this platform's own query set is in the tests, against the exact in-memory index.
|
|
1449
|
+
*/
|
|
1450
|
+
export const VECTOR_MIGRATIONS = [
|
|
1451
|
+
{
|
|
1452
|
+
id: "0017_knowledge_chunks",
|
|
1453
|
+
up: [
|
|
1454
|
+
// `WITH SCHEMA public` on purpose. `CREATE EXTENSION IF NOT EXISTS` skips when the extension exists
|
|
1455
|
+
// *anywhere*, so without a fixed schema the first caller decides where the `vector` type lives and every
|
|
1456
|
+
// later schema finds it invisible. Pinning it to `public` — which is on every normal deployment's
|
|
1457
|
+
// search_path — makes the type resolvable from any schema. Found by running this suite against a real
|
|
1458
|
+
// pgvector server, where each test gets its own schema.
|
|
1459
|
+
`CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA public`,
|
|
1460
|
+
`CREATE TABLE IF NOT EXISTS knowledge_chunks (
|
|
1461
|
+
tenant_id text NOT NULL,
|
|
1462
|
+
id text NOT NULL,
|
|
1463
|
+
source_type text NOT NULL,
|
|
1464
|
+
source_id text NOT NULL,
|
|
1465
|
+
chunk_index integer NOT NULL,
|
|
1466
|
+
content text NOT NULL,
|
|
1467
|
+
token_count integer NOT NULL,
|
|
1468
|
+
-- The authorisation subject, on the chunk. This is what lets permission filtering happen *inside* the
|
|
1469
|
+
-- query: filtering after retrieval leaks through result counts -- ask for ten, get three, and you have
|
|
1470
|
+
-- learned that seven exist you may not see.
|
|
1471
|
+
auth_subject text NOT NULL,
|
|
1472
|
+
embedding vector(${VECTOR_DIMENSIONS}) NOT NULL,
|
|
1473
|
+
embedding_model text NOT NULL,
|
|
1474
|
+
embedding_version text NOT NULL,
|
|
1475
|
+
embedding_dims integer NOT NULL,
|
|
1476
|
+
locator text,
|
|
1477
|
+
created_at timestamptz NOT NULL,
|
|
1478
|
+
PRIMARY KEY (tenant_id, id),
|
|
1479
|
+
UNIQUE (tenant_id, source_type, source_id, chunk_index),
|
|
1480
|
+
CHECK (chunk_index >= 0),
|
|
1481
|
+
CHECK (token_count >= 0),
|
|
1482
|
+
CHECK (embedding_dims = ${VECTOR_DIMENSIONS})
|
|
1483
|
+
)`,
|
|
1484
|
+
// Serves listBySource and replaceSource's delete, both of which scan by source.
|
|
1485
|
+
`CREATE INDEX IF NOT EXISTS knowledge_chunks_source_idx
|
|
1486
|
+
ON knowledge_chunks (tenant_id, source_type, source_id, chunk_index)`,
|
|
1487
|
+
// Serves listStaleSources: the work list for an incremental re-index.
|
|
1488
|
+
`CREATE INDEX IF NOT EXISTS knowledge_chunks_model_idx
|
|
1489
|
+
ON knowledge_chunks (tenant_id, embedding_model, embedding_version)`,
|
|
1490
|
+
// HNSW on cosine distance, matching the metric the ports normalise to. Built after the table so the
|
|
1491
|
+
// initial build is over an empty relation, which is fast; a build over a loaded table locks it.
|
|
1492
|
+
`CREATE INDEX IF NOT EXISTS knowledge_chunks_embedding_idx
|
|
1493
|
+
ON knowledge_chunks USING hnsw (embedding vector_cosine_ops)
|
|
1494
|
+
WITH (m = 16, ef_construction = 64)`,
|
|
1495
|
+
// #136. Full-text search over the same rows, so semantic and keyword retrieval share one
|
|
1496
|
+
// permission-filtered source of truth. A GIN index on a *generated* column rather than an expression
|
|
1497
|
+
// index: an expression index has to be written identically in every query to be used, and one query
|
|
1498
|
+
// spelled slightly differently silently sequential-scans -- which is a performance cliff nobody notices
|
|
1499
|
+
// until the corpus is large.
|
|
1500
|
+
//
|
|
1501
|
+
// `simple` rather than `english`, deliberately. The English configuration stems, so `renewals` and
|
|
1502
|
+
// `renewal` match -- which is what the *vector* signal is for. Keyword retrieval exists for exact terms
|
|
1503
|
+
// (`ERR-4021`, `Q3-2026`), and a stemmer is precisely what destroys them. Two signals with two jobs.
|
|
1504
|
+
`ALTER TABLE knowledge_chunks
|
|
1505
|
+
ADD COLUMN IF NOT EXISTS content_tsv tsvector
|
|
1506
|
+
GENERATED ALWAYS AS (to_tsvector('simple', content)) STORED`,
|
|
1507
|
+
`CREATE INDEX IF NOT EXISTS knowledge_chunks_fts_idx
|
|
1508
|
+
ON knowledge_chunks USING gin (content_tsv)`,
|
|
1509
|
+
],
|
|
1510
|
+
down: [
|
|
1511
|
+
`DROP INDEX IF EXISTS knowledge_chunks_fts_idx`,
|
|
1512
|
+
`DROP INDEX IF EXISTS knowledge_chunks_embedding_idx`,
|
|
1513
|
+
`DROP INDEX IF EXISTS knowledge_chunks_model_idx`,
|
|
1514
|
+
`DROP INDEX IF EXISTS knowledge_chunks_source_idx`,
|
|
1515
|
+
`DROP TABLE IF EXISTS knowledge_chunks`,
|
|
1516
|
+
],
|
|
1517
|
+
},
|
|
1518
|
+
];
|
|
1519
|
+
/** Whether this database can hold vectors, so a caller does not have to learn it from a failure. */
|
|
1520
|
+
export const hasVectorExtension = async (sql) => {
|
|
1521
|
+
try {
|
|
1522
|
+
const rows = await sql.query(`SELECT true AS ok FROM pg_available_extensions WHERE name = 'vector'`);
|
|
1523
|
+
return rows.length > 0;
|
|
1524
|
+
}
|
|
1525
|
+
catch {
|
|
1526
|
+
// A database that cannot answer the question cannot have the extension either.
|
|
1527
|
+
return false;
|
|
1528
|
+
}
|
|
1529
|
+
};
|
|
1530
|
+
/** Applies the vector migrations. Call only where `hasVectorExtension` is true. */
|
|
1531
|
+
export const migrateVector = async (sql) => {
|
|
1532
|
+
for (const m of VECTOR_MIGRATIONS)
|
|
1533
|
+
for (const stmt of m.up)
|
|
1534
|
+
await sql.query(stmt);
|
|
1535
|
+
};
|
|
1536
|
+
export const rollbackVector = async (sql) => {
|
|
1537
|
+
for (const m of [...VECTOR_MIGRATIONS].reverse())
|
|
1538
|
+
for (const stmt of m.down)
|
|
1539
|
+
await sql.query(stmt);
|
|
1540
|
+
};
|
|
1541
|
+
/**
|
|
1542
|
+
* The applied-migrations ledger. Defined here rather than in `schema.ts` because both provisioning
|
|
1543
|
+
* paths write it -- `migrate()` below and `SchemaManager.apply()` -- and a second copy of the DDL is
|
|
1544
|
+
* how they would drift. They did drift: `migrate()` used to apply statements without recording them,
|
|
1545
|
+
* so `/readyz`'s schema probe read an empty ledger and never reported ready for anyone who
|
|
1546
|
+
* provisioned with `migrate()` rather than `apply()`.
|
|
1547
|
+
*/
|
|
1548
|
+
export const MIGRATION_LEDGER = `CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
1549
|
+
id text PRIMARY KEY,
|
|
1550
|
+
applied_at timestamptz NOT NULL DEFAULT now()
|
|
1551
|
+
)`;
|
|
1552
|
+
/**
|
|
1553
|
+
* Ledger rows, or an empty set where the table does not exist yet. Read-only: asking what has been
|
|
1554
|
+
* applied must not itself apply anything.
|
|
1555
|
+
*/
|
|
1556
|
+
export const appliedMigrationIds = async (sql) => {
|
|
1557
|
+
try {
|
|
1558
|
+
const rows = await sql.query(`SELECT id FROM schema_migrations`);
|
|
1559
|
+
return new Set(rows.map((r) => r.id));
|
|
1560
|
+
}
|
|
1561
|
+
catch {
|
|
1562
|
+
return new Set();
|
|
1563
|
+
}
|
|
1564
|
+
};
|
|
1565
|
+
/**
|
|
1566
|
+
* Applies every pending migration and records it.
|
|
1567
|
+
*
|
|
1568
|
+
* Migrations already in the ledger are skipped, but a schema provisioned before the ledger existed
|
|
1569
|
+
* has the tables and no rows -- so the first run after upgrading re-executes all of them. That is
|
|
1570
|
+
* safe because every statement is idempotent (`migrations-rerunnable.test.ts` is what keeps it so),
|
|
1571
|
+
* and it leaves the ledger complete afterwards.
|
|
1572
|
+
*/
|
|
1573
|
+
export const migrate = async (sql) => {
|
|
1574
|
+
await sql.query(MIGRATION_LEDGER);
|
|
1575
|
+
const done = await appliedMigrationIds(sql);
|
|
1576
|
+
for (const m of MIGRATIONS) {
|
|
1577
|
+
if (done.has(m.id))
|
|
1578
|
+
continue;
|
|
1579
|
+
for (const stmt of m.up)
|
|
1580
|
+
await sql.query(stmt);
|
|
1581
|
+
// ON CONFLICT DO NOTHING: two processes migrating at once must not both record the same id.
|
|
1582
|
+
await sql.query(`INSERT INTO schema_migrations (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, [m.id]);
|
|
1583
|
+
}
|
|
1584
|
+
};
|
|
1585
|
+
export const rollback = async (sql) => {
|
|
1586
|
+
for (const m of [...MIGRATIONS].reverse()) {
|
|
1587
|
+
for (const stmt of m.down)
|
|
1588
|
+
await sql.query(stmt);
|
|
1589
|
+
// Forget the migration only once its `down` has succeeded, so a failure mid-rollback leaves a
|
|
1590
|
+
// ledger that still names what is actually present.
|
|
1591
|
+
await sql.query(`DELETE FROM schema_migrations WHERE id = $1`, [m.id]).catch(() => undefined);
|
|
1592
|
+
}
|
|
1593
|
+
};
|
|
1594
|
+
//# sourceMappingURL=migrations.js.map
|