@retinue/agentkit 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/README.md +310 -0
- package/dist/adapters/bullmq/consumer.d.ts +33 -0
- package/dist/adapters/bullmq/consumer.js +41 -0
- package/dist/adapters/bullmq/dispatcher.d.ts +74 -0
- package/dist/adapters/bullmq/dispatcher.js +160 -0
- package/dist/adapters/bullmq/export.d.ts +31 -0
- package/dist/adapters/bullmq/export.js +53 -0
- package/dist/adapters/bullmq/extraction.d.ts +42 -0
- package/dist/adapters/bullmq/extraction.js +63 -0
- package/dist/adapters/bullmq/index.d.ts +13 -0
- package/dist/adapters/bullmq/index.js +13 -0
- package/dist/adapters/bullmq/lock.d.ts +77 -0
- package/dist/adapters/bullmq/lock.js +126 -0
- package/dist/adapters/bullmq/queue.d.ts +50 -0
- package/dist/adapters/bullmq/queue.js +81 -0
- package/dist/adapters/memory/artifact-exports.d.ts +11 -0
- package/dist/adapters/memory/artifact-exports.js +102 -0
- package/dist/adapters/memory/artifacts.d.ts +15 -0
- package/dist/adapters/memory/artifacts.js +134 -0
- package/dist/adapters/memory/blobs.d.ts +7 -0
- package/dist/adapters/memory/blobs.js +27 -0
- package/dist/adapters/memory/evaluation.d.ts +18 -0
- package/dist/adapters/memory/evaluation.js +148 -0
- package/dist/adapters/memory/files.d.ts +27 -0
- package/dist/adapters/memory/files.js +0 -0
- package/dist/adapters/memory/flows.d.ts +16 -0
- package/dist/adapters/memory/flows.js +117 -0
- package/dist/adapters/memory/hitl.d.ts +9 -0
- package/dist/adapters/memory/hitl.js +130 -0
- package/dist/adapters/memory/idempotency.d.ts +13 -0
- package/dist/adapters/memory/idempotency.js +32 -0
- package/dist/adapters/memory/index.d.ts +39 -0
- package/dist/adapters/memory/index.js +107 -0
- package/dist/adapters/memory/knowledge.d.ts +43 -0
- package/dist/adapters/memory/knowledge.js +248 -0
- package/dist/adapters/memory/mcp.d.ts +9 -0
- package/dist/adapters/memory/mcp.js +37 -0
- package/dist/adapters/memory/message-store.d.ts +17 -0
- package/dist/adapters/memory/message-store.js +70 -0
- package/dist/adapters/memory/principal-memory.d.ts +7 -0
- package/dist/adapters/memory/principal-memory.js +83 -0
- package/dist/adapters/memory/runtime.d.ts +29 -0
- package/dist/adapters/memory/runtime.js +0 -0
- package/dist/adapters/memory/sessions.d.ts +29 -0
- package/dist/adapters/memory/sessions.js +0 -0
- package/dist/adapters/memory/skills.d.ts +10 -0
- package/dist/adapters/memory/skills.js +41 -0
- package/dist/adapters/memory/thread-summary.d.ts +7 -0
- package/dist/adapters/memory/thread-summary.js +29 -0
- package/dist/adapters/memory/usage-limits.d.ts +13 -0
- package/dist/adapters/memory/usage-limits.js +72 -0
- package/dist/adapters/memory/usage.d.ts +16 -0
- package/dist/adapters/memory/usage.js +279 -0
- package/dist/adapters/otel/index.d.ts +111 -0
- package/dist/adapters/otel/index.js +133 -0
- package/dist/adapters/postgres/artifact-exports.d.ts +12 -0
- package/dist/adapters/postgres/artifact-exports.js +117 -0
- package/dist/adapters/postgres/artifacts.d.ts +16 -0
- package/dist/adapters/postgres/artifacts.js +172 -0
- package/dist/adapters/postgres/checkpoint-store.d.ts +16 -0
- package/dist/adapters/postgres/checkpoint-store.js +34 -0
- package/dist/adapters/postgres/config.d.ts +15 -0
- package/dist/adapters/postgres/config.js +187 -0
- package/dist/adapters/postgres/conversation-store.d.ts +4 -0
- package/dist/adapters/postgres/conversation-store.js +82 -0
- package/dist/adapters/postgres/evaluation.d.ts +17 -0
- package/dist/adapters/postgres/evaluation.js +193 -0
- package/dist/adapters/postgres/file-content.d.ts +30 -0
- package/dist/adapters/postgres/file-content.js +111 -0
- package/dist/adapters/postgres/files.d.ts +19 -0
- package/dist/adapters/postgres/files.js +209 -0
- package/dist/adapters/postgres/flows.d.ts +20 -0
- package/dist/adapters/postgres/flows.js +206 -0
- package/dist/adapters/postgres/hitl.d.ts +5 -0
- package/dist/adapters/postgres/hitl.js +247 -0
- package/dist/adapters/postgres/index.d.ts +35 -0
- package/dist/adapters/postgres/index.js +35 -0
- package/dist/adapters/postgres/knowledge.d.ts +48 -0
- package/dist/adapters/postgres/knowledge.js +255 -0
- package/dist/adapters/postgres/memory.d.ts +14 -0
- package/dist/adapters/postgres/memory.js +194 -0
- package/dist/adapters/postgres/message-store.d.ts +11 -0
- package/dist/adapters/postgres/message-store.js +145 -0
- package/dist/adapters/postgres/migrations.d.ts +69 -0
- package/dist/adapters/postgres/migrations.js +1594 -0
- package/dist/adapters/postgres/pg-executor.d.ts +19 -0
- package/dist/adapters/postgres/pg-executor.js +32 -0
- package/dist/adapters/postgres/retention.d.ts +26 -0
- package/dist/adapters/postgres/retention.js +59 -0
- package/dist/adapters/postgres/rollups.d.ts +17 -0
- package/dist/adapters/postgres/rollups.js +267 -0
- package/dist/adapters/postgres/run-coordinator.d.ts +5 -0
- package/dist/adapters/postgres/run-coordinator.js +98 -0
- package/dist/adapters/postgres/run-event-log.d.ts +26 -0
- package/dist/adapters/postgres/run-event-log.js +30 -0
- package/dist/adapters/postgres/run-store.d.ts +4 -0
- package/dist/adapters/postgres/run-store.js +199 -0
- package/dist/adapters/postgres/schema.d.ts +39 -0
- package/dist/adapters/postgres/schema.js +70 -0
- package/dist/adapters/postgres/session-state.d.ts +7 -0
- package/dist/adapters/postgres/session-state.js +99 -0
- package/dist/adapters/postgres/sql.d.ts +8 -0
- package/dist/adapters/postgres/sql.js +2 -0
- package/dist/adapters/postgres/transaction.d.ts +37 -0
- package/dist/adapters/postgres/transaction.js +93 -0
- package/dist/adapters/postgres/unit-of-work.d.ts +18 -0
- package/dist/adapters/postgres/unit-of-work.js +8 -0
- package/dist/adapters/postgres/usage-limits.d.ts +15 -0
- package/dist/adapters/postgres/usage-limits.js +136 -0
- package/dist/adapters/postgres/usage.d.ts +15 -0
- package/dist/adapters/postgres/usage.js +226 -0
- package/dist/adapters/redis/index.d.ts +9 -0
- package/dist/adapters/redis/index.js +9 -0
- package/dist/adapters/redis/realtime.d.ts +74 -0
- package/dist/adapters/redis/realtime.js +112 -0
- package/dist/adapters/supabase/index.d.ts +88 -0
- package/dist/adapters/supabase/index.js +84 -0
- package/dist/adapters/supabase/realtime.d.ts +30 -0
- package/dist/adapters/supabase/realtime.js +53 -0
- package/dist/adapters/supabase/rls.d.ts +99 -0
- package/dist/adapters/supabase/rls.js +216 -0
- package/dist/adapters/supabase/storage.d.ts +50 -0
- package/dist/adapters/supabase/storage.js +207 -0
- package/dist/agents/agent.d.ts +66 -0
- package/dist/agents/agent.js +209 -0
- package/dist/agents/define.d.ts +21 -0
- package/dist/agents/define.js +63 -0
- package/dist/agents/engine.d.ts +98 -0
- package/dist/agents/engine.js +462 -0
- package/dist/agents/index.d.ts +50 -0
- package/dist/agents/index.js +17 -0
- package/dist/artifacts/index.d.ts +114 -0
- package/dist/artifacts/index.js +219 -0
- package/dist/authorization/index.d.ts +76 -0
- package/dist/authorization/index.js +76 -0
- package/dist/capabilities/index.d.ts +120 -0
- package/dist/capabilities/index.js +167 -0
- package/dist/capabilities/runtime.d.ts +89 -0
- package/dist/capabilities/runtime.js +84 -0
- package/dist/citations/index.d.ts +161 -0
- package/dist/citations/index.js +182 -0
- package/dist/context/assembler.d.ts +82 -0
- package/dist/context/assembler.js +129 -0
- package/dist/context/compaction.d.ts +45 -0
- package/dist/context/compaction.js +55 -0
- package/dist/context/index.d.ts +75 -0
- package/dist/context/index.js +17 -0
- package/dist/core/content-parts.d.ts +194 -0
- package/dist/core/content-parts.js +23 -0
- package/dist/core/context.d.ts +51 -0
- package/dist/core/context.js +9 -0
- package/dist/core/env.d.ts +25 -0
- package/dist/core/env.js +41 -0
- package/dist/core/errors.d.ts +30 -0
- package/dist/core/errors.js +65 -0
- package/dist/core/events.d.ts +139 -0
- package/dist/core/events.js +99 -0
- package/dist/core/ids.d.ts +52 -0
- package/dist/core/ids.js +13 -0
- package/dist/core/index.d.ts +9 -0
- package/dist/core/index.js +9 -0
- package/dist/core/tokens.d.ts +22 -0
- package/dist/core/tokens.js +22 -0
- package/dist/core/validation.d.ts +34 -0
- package/dist/core/validation.js +176 -0
- package/dist/documents/extraction.d.ts +121 -0
- package/dist/documents/extraction.js +293 -0
- package/dist/documents/index.d.ts +199 -0
- package/dist/documents/index.js +65 -0
- package/dist/documents/parsers/pdf.d.ts +47 -0
- package/dist/documents/parsers/pdf.js +508 -0
- package/dist/documents/parsers/text.d.ts +59 -0
- package/dist/documents/parsers/text.js +325 -0
- package/dist/documents/read-tool.d.ts +52 -0
- package/dist/documents/read-tool.js +109 -0
- package/dist/documents/render.d.ts +29 -0
- package/dist/documents/render.js +59 -0
- package/dist/documents/vision.d.ts +159 -0
- package/dist/documents/vision.js +214 -0
- package/dist/entries/adapters-bullmq.d.ts +8 -0
- package/dist/entries/adapters-bullmq.js +8 -0
- package/dist/entries/adapters-otel.d.ts +13 -0
- package/dist/entries/adapters-otel.js +13 -0
- package/dist/entries/adapters-postgres.d.ts +10 -0
- package/dist/entries/adapters-postgres.js +10 -0
- package/dist/entries/adapters-redis.d.ts +3 -0
- package/dist/entries/adapters-redis.js +3 -0
- package/dist/entries/context.d.ts +20 -0
- package/dist/entries/context.js +20 -0
- package/dist/entries/flows.d.ts +15 -0
- package/dist/entries/flows.js +15 -0
- package/dist/entries/hitl.d.ts +10 -0
- package/dist/entries/hitl.js +10 -0
- package/dist/entries/knowledge.d.ts +18 -0
- package/dist/entries/knowledge.js +19 -0
- package/dist/entries/mcp.d.ts +10 -0
- package/dist/entries/mcp.js +10 -0
- package/dist/entries/observability.d.ts +14 -0
- package/dist/entries/observability.js +16 -0
- package/dist/entries/persistence.d.ts +11 -0
- package/dist/entries/persistence.js +11 -0
- package/dist/entries/providers.d.ts +14 -0
- package/dist/entries/providers.js +14 -0
- package/dist/entries/runtime.d.ts +13 -0
- package/dist/entries/runtime.js +13 -0
- package/dist/entries/server.d.ts +24 -0
- package/dist/entries/server.js +24 -0
- package/dist/entries/tools.d.ts +21 -0
- package/dist/entries/tools.js +21 -0
- package/dist/entries/usage.d.ts +10 -0
- package/dist/entries/usage.js +10 -0
- package/dist/evaluation/gate.d.ts +168 -0
- package/dist/evaluation/gate.js +180 -0
- package/dist/evaluation/graders.d.ts +125 -0
- package/dist/evaluation/graders.js +203 -0
- package/dist/evaluation/index.d.ts +120 -0
- package/dist/evaluation/index.js +183 -0
- package/dist/evaluation/judge.d.ts +75 -0
- package/dist/evaluation/judge.js +111 -0
- package/dist/export/index.d.ts +162 -0
- package/dist/export/index.js +363 -0
- package/dist/export/markdown.d.ts +19 -0
- package/dist/export/markdown.js +29 -0
- package/dist/export/pdf.d.ts +73 -0
- package/dist/export/pdf.js +407 -0
- package/dist/files/context.d.ts +97 -0
- package/dist/files/context.js +185 -0
- package/dist/files/index.d.ts +210 -0
- package/dist/files/index.js +338 -0
- package/dist/files/read-tool.d.ts +81 -0
- package/dist/files/read-tool.js +163 -0
- package/dist/files/turn-parts.d.ts +96 -0
- package/dist/files/turn-parts.js +171 -0
- package/dist/flows/index.d.ts +270 -0
- package/dist/flows/index.js +62 -0
- package/dist/flows/interpreter.d.ts +146 -0
- package/dist/flows/interpreter.js +426 -0
- package/dist/flows/runner.d.ts +145 -0
- package/dist/flows/runner.js +270 -0
- package/dist/graphql/index.d.ts +8 -0
- package/dist/graphql/index.js +8 -0
- package/dist/graphql/resolvers.d.ts +237 -0
- package/dist/graphql/resolvers.js +253 -0
- package/dist/graphql/schema.d.ts +11 -0
- package/dist/graphql/schema.js +258 -0
- package/dist/graphql/sse.d.ts +77 -0
- package/dist/graphql/sse.js +100 -0
- package/dist/hitl/approved-execution.d.ts +127 -0
- package/dist/hitl/approved-execution.js +177 -0
- package/dist/hitl/index.d.ts +79 -0
- package/dist/hitl/index.js +12 -0
- package/dist/hitl/service.d.ts +221 -0
- package/dist/hitl/service.js +268 -0
- package/dist/idempotency/index.d.ts +70 -0
- package/dist/idempotency/index.js +59 -0
- package/dist/index.d.ts +103 -0
- package/dist/index.js +60 -0
- package/dist/knowledge/chunking.d.ts +57 -0
- package/dist/knowledge/chunking.js +158 -0
- package/dist/knowledge/index.d.ts +119 -0
- package/dist/knowledge/index.js +166 -0
- package/dist/knowledge/retrieval.d.ts +146 -0
- package/dist/knowledge/retrieval.js +170 -0
- package/dist/loadtest/harness.d.ts +168 -0
- package/dist/loadtest/harness.js +507 -0
- package/dist/loadtest/index.d.ts +13 -0
- package/dist/loadtest/index.js +13 -0
- package/dist/loadtest/injection.d.ts +89 -0
- package/dist/loadtest/injection.js +147 -0
- package/dist/loadtest/metrics.d.ts +197 -0
- package/dist/loadtest/metrics.js +160 -0
- package/dist/loadtest/runbooks.d.ts +28 -0
- package/dist/loadtest/runbooks.js +159 -0
- package/dist/loadtest/scenario.d.ts +104 -0
- package/dist/loadtest/scenario.js +208 -0
- package/dist/mcp/egress.d.ts +53 -0
- package/dist/mcp/egress.js +115 -0
- package/dist/mcp/index.d.ts +93 -0
- package/dist/mcp/index.js +33 -0
- package/dist/mcp/provider.d.ts +62 -0
- package/dist/mcp/provider.js +0 -0
- package/dist/models/index.d.ts +98 -0
- package/dist/models/index.js +74 -0
- package/dist/models/pricing.d.ts +24 -0
- package/dist/models/pricing.js +37 -0
- package/dist/models/provider-factory.d.ts +31 -0
- package/dist/models/provider-factory.js +67 -0
- package/dist/models/streaming.d.ts +145 -0
- package/dist/models/streaming.js +272 -0
- package/dist/models/vision.d.ts +38 -0
- package/dist/models/vision.js +62 -0
- package/dist/persistence/index.d.ts +1654 -0
- package/dist/persistence/index.js +226 -0
- package/dist/principal-memory/index.d.ts +106 -0
- package/dist/principal-memory/index.js +89 -0
- package/dist/retention/index.d.ts +89 -0
- package/dist/retention/index.js +70 -0
- package/dist/runtime/checkpoint.d.ts +37 -0
- package/dist/runtime/checkpoint.js +22 -0
- package/dist/runtime/index.d.ts +118 -0
- package/dist/runtime/index.js +69 -0
- package/dist/runtime/retry.d.ts +95 -0
- package/dist/runtime/retry.js +126 -0
- package/dist/runtime/serialization.d.ts +85 -0
- package/dist/runtime/serialization.js +95 -0
- package/dist/runtime/streaming.d.ts +54 -0
- package/dist/runtime/streaming.js +115 -0
- package/dist/runtime/worker.d.ts +130 -0
- package/dist/runtime/worker.js +405 -0
- package/dist/security/checklist.d.ts +53 -0
- package/dist/security/checklist.js +204 -0
- package/dist/security/findings.d.ts +56 -0
- package/dist/security/findings.js +168 -0
- package/dist/security/index.d.ts +14 -0
- package/dist/security/index.js +14 -0
- package/dist/security/prompt-safety.d.ts +100 -0
- package/dist/security/prompt-safety.js +133 -0
- package/dist/server/boot.d.ts +32 -0
- package/dist/server/boot.js +36 -0
- package/dist/server/cli-worker.d.ts +37 -0
- package/dist/server/cli-worker.js +151 -0
- package/dist/server/cli.d.ts +27 -0
- package/dist/server/cli.js +74 -0
- package/dist/server/config.d.ts +42 -0
- package/dist/server/config.js +127 -0
- package/dist/server/health.d.ts +59 -0
- package/dist/server/health.js +90 -0
- package/dist/server/host.d.ts +39 -0
- package/dist/server/host.js +124 -0
- package/dist/server/index.d.ts +15 -0
- package/dist/server/index.js +15 -0
- package/dist/server/main.d.ts +16 -0
- package/dist/server/main.js +31 -0
- package/dist/server/sse-route.d.ts +21 -0
- package/dist/server/sse-route.js +282 -0
- package/dist/skills/index.d.ts +67 -0
- package/dist/skills/index.js +31 -0
- package/dist/skills/resolver.d.ts +54 -0
- package/dist/skills/resolver.js +121 -0
- package/dist/teams/index.d.ts +93 -0
- package/dist/teams/index.js +207 -0
- package/dist/telemetry/index.d.ts +157 -0
- package/dist/telemetry/index.js +71 -0
- package/dist/telemetry/instrument.d.ts +108 -0
- package/dist/telemetry/instrument.js +232 -0
- package/dist/telemetry/log-events.d.ts +17 -0
- package/dist/telemetry/log-events.js +58 -0
- package/dist/telemetry/metrics.d.ts +123 -0
- package/dist/telemetry/metrics.js +135 -0
- package/dist/telemetry/noop.d.ts +39 -0
- package/dist/telemetry/noop.js +143 -0
- package/dist/telemetry/redaction.d.ts +64 -0
- package/dist/telemetry/redaction.js +153 -0
- package/dist/telemetry/spans.d.ts +56 -0
- package/dist/telemetry/spans.js +78 -0
- package/dist/telemetry/trace-context.d.ts +55 -0
- package/dist/telemetry/trace-context.js +60 -0
- package/dist/toolkit/compute.d.ts +53 -0
- package/dist/toolkit/compute.js +152 -0
- package/dist/toolkit/data.d.ts +98 -0
- package/dist/toolkit/data.js +235 -0
- package/dist/toolkit/http.d.ts +113 -0
- package/dist/toolkit/http.js +205 -0
- package/dist/toolkit/index.d.ts +21 -0
- package/dist/toolkit/index.js +17 -0
- package/dist/toolkit/web.d.ts +107 -0
- package/dist/toolkit/web.js +147 -0
- package/dist/tools/define.d.ts +25 -0
- package/dist/tools/define.js +45 -0
- package/dist/tools/delegating.d.ts +132 -0
- package/dist/tools/delegating.js +211 -0
- package/dist/tools/index.d.ts +129 -0
- package/dist/tools/index.js +33 -0
- package/dist/tools/library/compute.d.ts +11 -0
- package/dist/tools/library/compute.js +46 -0
- package/dist/tools/library/data.d.ts +16 -0
- package/dist/tools/library/data.js +92 -0
- package/dist/tools/library/http.d.ts +28 -0
- package/dist/tools/library/http.js +71 -0
- package/dist/tools/library/index.d.ts +97 -0
- package/dist/tools/library/index.js +134 -0
- package/dist/tools/library/knowledge.d.ts +39 -0
- package/dist/tools/library/knowledge.js +58 -0
- package/dist/tools/library/web.d.ts +19 -0
- package/dist/tools/library/web.js +65 -0
- package/dist/tools/meta-tools.d.ts +19 -0
- package/dist/tools/meta-tools.js +36 -0
- package/dist/tools/registry.d.ts +146 -0
- package/dist/tools/registry.js +291 -0
- package/dist/usage/index.d.ts +105 -0
- package/dist/usage/index.js +20 -0
- package/dist/usage/quota.d.ts +258 -0
- package/dist/usage/quota.js +510 -0
- package/dist/usage/recorder.d.ts +29 -0
- package/dist/usage/recorder.js +96 -0
- package/dist/usage/rollups.d.ts +121 -0
- package/dist/usage/rollups.js +157 -0
- package/dist/worker/export.d.ts +57 -0
- package/dist/worker/export.js +81 -0
- package/dist/worker/extraction.d.ts +57 -0
- package/dist/worker/extraction.js +84 -0
- package/dist/worker/main.d.ts +103 -0
- package/dist/worker/main.js +159 -0
- package/package.json +187 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Production `SqlExecutor` backed by node-postgres. `pg` is imported type-only here — the caller
|
|
3
|
+
* supplies the `Pool` — so the package has no runtime coupling to a specific driver instance.
|
|
4
|
+
*/
|
|
5
|
+
import type { Pool } from "pg";
|
|
6
|
+
import type { SqlExecutor } from "./sql.js";
|
|
7
|
+
import type { ConnectionOpener } from "./transaction.js";
|
|
8
|
+
export declare const createPgExecutor: (pool: Pool) => SqlExecutor;
|
|
9
|
+
/**
|
|
10
|
+
* Checks out one connection for the duration of `fn` — the primitive `createPgExecutor` cannot
|
|
11
|
+
* provide, because `pool.query` picks a different connection per call, so `BEGIN` / work / `COMMIT`
|
|
12
|
+
* through it would land on three connections and guarantee nothing (#98).
|
|
13
|
+
*
|
|
14
|
+
* `searchPath` exists for the conformance suite, which isolates each executor in its own schema. A
|
|
15
|
+
* pooled connection carries whatever `search_path` its last user left, so it has to be set per
|
|
16
|
+
* checkout rather than once at pool creation.
|
|
17
|
+
*/
|
|
18
|
+
export declare const createPoolOpener: (pool: Pool, searchPath?: string) => ConnectionOpener;
|
|
19
|
+
//# sourceMappingURL=pg-executor.d.ts.map
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export const createPgExecutor = (pool) => ({
|
|
2
|
+
query(text, params) {
|
|
3
|
+
return pool.query(text, params ? [...params] : undefined).then((r) => r.rows);
|
|
4
|
+
},
|
|
5
|
+
});
|
|
6
|
+
/**
|
|
7
|
+
* Checks out one connection for the duration of `fn` — the primitive `createPgExecutor` cannot
|
|
8
|
+
* provide, because `pool.query` picks a different connection per call, so `BEGIN` / work / `COMMIT`
|
|
9
|
+
* through it would land on three connections and guarantee nothing (#98).
|
|
10
|
+
*
|
|
11
|
+
* `searchPath` exists for the conformance suite, which isolates each executor in its own schema. A
|
|
12
|
+
* pooled connection carries whatever `search_path` its last user left, so it has to be set per
|
|
13
|
+
* checkout rather than once at pool creation.
|
|
14
|
+
*/
|
|
15
|
+
export const createPoolOpener = (pool, searchPath) => {
|
|
16
|
+
return async (fn) => {
|
|
17
|
+
const client = await pool.connect();
|
|
18
|
+
try {
|
|
19
|
+
if (searchPath)
|
|
20
|
+
await client.query(`SET search_path TO ${searchPath}`);
|
|
21
|
+
return await fn({
|
|
22
|
+
query(text, params) {
|
|
23
|
+
return client.query(text, params ? [...params] : undefined).then((r) => r.rows);
|
|
24
|
+
},
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
finally {
|
|
28
|
+
client.release();
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
};
|
|
32
|
+
//# sourceMappingURL=pg-executor.js.map
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Postgres `RunEventPruner` (#151).
|
|
3
|
+
*
|
|
4
|
+
* One statement, and every interesting decision is in it.
|
|
5
|
+
*
|
|
6
|
+
* **`ctid IN (SELECT … LIMIT n)`.** The bound has to be on the *rows selected*, not on the delete, and Postgres
|
|
7
|
+
* has no `DELETE … LIMIT`. Selecting `ctid` — the physical row locator — lets the subquery do a bounded,
|
|
8
|
+
* index-driven scan and the delete touch exactly those rows. The alternative shapes are worse: a correlated
|
|
9
|
+
* `IN (tenant_id, run_id, sequence)` tuple list is a much larger comparison per row, and a CTE with `RETURNING`
|
|
10
|
+
* still needs the same bounded selection to feed it.
|
|
11
|
+
*
|
|
12
|
+
* **The safety join is inside the subquery.** `run_events` has no status of its own, so "is this run terminal?"
|
|
13
|
+
* is a lookup on `runs` by `(tenant_id, run_id)` — which is `runs`' primary key, so it costs an index probe per
|
|
14
|
+
* candidate and no extra index. Putting the join in the subquery means a non-terminal run's rows are never
|
|
15
|
+
* *selected*, so they cannot be deleted by a later mistake in the outer statement.
|
|
16
|
+
*
|
|
17
|
+
* **`FOR UPDATE SKIP LOCKED` is deliberately absent.** Two concurrent prunes selecting overlapping `ctid`s is
|
|
18
|
+
* harmless: the second `DELETE` matches no row for the ones already gone and reports a smaller count. Adding row
|
|
19
|
+
* locks would serialise the sweeps and, worse, hold locks across the join — which is exactly the blocking this
|
|
20
|
+
* batching exists to avoid. Idempotency here comes from the delete being a no-op on an absent row, not from
|
|
21
|
+
* exclusion.
|
|
22
|
+
*/
|
|
23
|
+
import { type RunEventPruner } from "../../retention/index.js";
|
|
24
|
+
import type { SqlExecutor } from "./sql.js";
|
|
25
|
+
export declare const createPostgresRunEventPruner: (sql: SqlExecutor) => RunEventPruner;
|
|
26
|
+
//# sourceMappingURL=retention.d.ts.map
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Postgres `RunEventPruner` (#151).
|
|
3
|
+
*
|
|
4
|
+
* One statement, and every interesting decision is in it.
|
|
5
|
+
*
|
|
6
|
+
* **`ctid IN (SELECT … LIMIT n)`.** The bound has to be on the *rows selected*, not on the delete, and Postgres
|
|
7
|
+
* has no `DELETE … LIMIT`. Selecting `ctid` — the physical row locator — lets the subquery do a bounded,
|
|
8
|
+
* index-driven scan and the delete touch exactly those rows. The alternative shapes are worse: a correlated
|
|
9
|
+
* `IN (tenant_id, run_id, sequence)` tuple list is a much larger comparison per row, and a CTE with `RETURNING`
|
|
10
|
+
* still needs the same bounded selection to feed it.
|
|
11
|
+
*
|
|
12
|
+
* **The safety join is inside the subquery.** `run_events` has no status of its own, so "is this run terminal?"
|
|
13
|
+
* is a lookup on `runs` by `(tenant_id, run_id)` — which is `runs`' primary key, so it costs an index probe per
|
|
14
|
+
* candidate and no extra index. Putting the join in the subquery means a non-terminal run's rows are never
|
|
15
|
+
* *selected*, so they cannot be deleted by a later mistake in the outer statement.
|
|
16
|
+
*
|
|
17
|
+
* **`FOR UPDATE SKIP LOCKED` is deliberately absent.** Two concurrent prunes selecting overlapping `ctid`s is
|
|
18
|
+
* harmless: the second `DELETE` matches no row for the ones already gone and reports a smaller count. Adding row
|
|
19
|
+
* locks would serialise the sweeps and, worse, hold locks across the join — which is exactly the blocking this
|
|
20
|
+
* batching exists to avoid. Idempotency here comes from the delete being a no-op on an absent row, not from
|
|
21
|
+
* exclusion.
|
|
22
|
+
*/
|
|
23
|
+
import { PRUNABLE_RUN_STATUSES } from "../../retention/index.js";
|
|
24
|
+
/**
|
|
25
|
+
* The terminal statuses as a SQL list.
|
|
26
|
+
*
|
|
27
|
+
* Built from the exported constant so the SQL and the documented rule cannot drift. Quoted here rather than
|
|
28
|
+
* passed as a parameter because an array parameter would make the planner treat it as opaque and lose the
|
|
29
|
+
* index probe on `runs`; the values come from a closed list in our own source, never from input.
|
|
30
|
+
*/
|
|
31
|
+
const TERMINAL_LIST = PRUNABLE_RUN_STATUSES.map((s) => `'${s}'`).join(", ");
|
|
32
|
+
export const createPostgresRunEventPruner = (sql) => ({
|
|
33
|
+
async prune({ olderThan, limit }) {
|
|
34
|
+
// A zero or negative limit would make `ctid IN (SELECT ... LIMIT 0)` delete nothing, which is correct but
|
|
35
|
+
// costs a scan. Answered without touching the database, because a maintenance loop with a misconfigured
|
|
36
|
+
// batch size should not generate load.
|
|
37
|
+
if (limit <= 0)
|
|
38
|
+
return { deleted: 0 };
|
|
39
|
+
const rows = await sql.query(`DELETE FROM run_events
|
|
40
|
+
WHERE ctid IN (
|
|
41
|
+
SELECT e.ctid
|
|
42
|
+
FROM run_events e
|
|
43
|
+
JOIN runs r ON r.tenant_id = e.tenant_id AND r.id = e.run_id
|
|
44
|
+
WHERE e.created_at < $1::timestamptz
|
|
45
|
+
-- AC-2. A non-terminal run can still be reconciled against its log, so its events are never
|
|
46
|
+
-- selected however old they are -- including a run that has waited months on a human approval.
|
|
47
|
+
AND r.status IN (${TERMINAL_LIST})
|
|
48
|
+
-- Oldest first, so a bounded sweep makes progress at the end of the table that will never come back
|
|
49
|
+
-- rather than nibbling at whatever the scan happened to reach.
|
|
50
|
+
ORDER BY e.created_at
|
|
51
|
+
LIMIT $2
|
|
52
|
+
)
|
|
53
|
+
RETURNING ctid`, [olderThan, limit]);
|
|
54
|
+
// The row count from `RETURNING`, not from a driver-specific `rowCount`: the executor port returns rows, and
|
|
55
|
+
// counting them is the one form that is identical across every adapter.
|
|
56
|
+
return { deleted: rows.length };
|
|
57
|
+
},
|
|
58
|
+
});
|
|
59
|
+
//# sourceMappingURL=retention.js.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Postgres `UsageRollupStore` (#139).
|
|
3
|
+
*
|
|
4
|
+
* Two properties are in the SQL rather than above it.
|
|
5
|
+
*
|
|
6
|
+
* **`rebuild` is one statement.** It aggregates the bucket's raw events and upserts the row, so the read and
|
|
7
|
+
* the write cannot see different data — a read-then-write would let an event land between them and be lost from
|
|
8
|
+
* the rollup while sitting in the ledger, which is the "no lost records" half of AC-4.
|
|
9
|
+
*
|
|
10
|
+
* **Idempotency is the primary key.** `ON CONFLICT (tenant_id, period, bucket_start) DO UPDATE` means a
|
|
11
|
+
* re-run replaces rather than accumulates, and two workers racing one bucket write the same value. No applied-
|
|
12
|
+
* event set, nothing to keep exactly right forever, and no way to double count.
|
|
13
|
+
*/
|
|
14
|
+
import type { UsageRollupStore } from "../../persistence/index.js";
|
|
15
|
+
import type { SqlExecutor } from "./sql.js";
|
|
16
|
+
export declare const createPostgresUsageRollupStore: (sql: SqlExecutor) => UsageRollupStore;
|
|
17
|
+
//# sourceMappingURL=rollups.d.ts.map
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Postgres `UsageRollupStore` (#139).
|
|
3
|
+
*
|
|
4
|
+
* Two properties are in the SQL rather than above it.
|
|
5
|
+
*
|
|
6
|
+
* **`rebuild` is one statement.** It aggregates the bucket's raw events and upserts the row, so the read and
|
|
7
|
+
* the write cannot see different data — a read-then-write would let an event land between them and be lost from
|
|
8
|
+
* the rollup while sitting in the ledger, which is the "no lost records" half of AC-4.
|
|
9
|
+
*
|
|
10
|
+
* **Idempotency is the primary key.** `ON CONFLICT (tenant_id, period, bucket_start) DO UPDATE` means a
|
|
11
|
+
* re-run replaces rather than accumulates, and two workers racing one bucket write the same value. No applied-
|
|
12
|
+
* event set, nothing to keep exactly right forever, and no way to double count.
|
|
13
|
+
*/
|
|
14
|
+
const iso = (v) => (v instanceof Date ? v.toISOString() : v);
|
|
15
|
+
/**
|
|
16
|
+
* `bigint` comes back as a string from node-postgres — correct past 2^53 and wrong for arithmetic.
|
|
17
|
+
*
|
|
18
|
+
* Narrowed here because every one of these is well inside the safe range: a tenant would need 9×10^15 tokens
|
|
19
|
+
* in one hour to overflow. Left as strings, a caller adding two rollups would concatenate them.
|
|
20
|
+
*/
|
|
21
|
+
const toRollup = (r) => ({
|
|
22
|
+
// #175: the grain, so a caller holding a mixed list can tell a tenant total from one person's.
|
|
23
|
+
...(r.principal_id === null || r.principal_id === undefined
|
|
24
|
+
? {}
|
|
25
|
+
: { principalId: r.principal_id }),
|
|
26
|
+
period: r.period,
|
|
27
|
+
bucketStart: iso(r.bucket_start),
|
|
28
|
+
inputTokens: Number(r.input_tokens),
|
|
29
|
+
outputTokens: Number(r.output_tokens),
|
|
30
|
+
cachedInputTokens: Number(r.cached_input_tokens),
|
|
31
|
+
reasoningTokens: Number(r.reasoning_tokens),
|
|
32
|
+
costMinorUnits: Number(r.cost_minor_units),
|
|
33
|
+
eventCount: Number(r.event_count),
|
|
34
|
+
currency: r.currency,
|
|
35
|
+
computedAt: iso(r.computed_at),
|
|
36
|
+
});
|
|
37
|
+
const COLUMNS = `principal_id, period, bucket_start, input_tokens, output_tokens, cached_input_tokens,
|
|
38
|
+
reasoning_tokens, cost_minor_units, event_count, currency, computed_at`;
|
|
39
|
+
/** The interval a period spans, as a SQL literal. Two values, both from a closed union — never user input. */
|
|
40
|
+
/**
|
|
41
|
+
* The width of a bucket, as a SQL interval — total over `RollupPeriod`.
|
|
42
|
+
*
|
|
43
|
+
* A `Record` rather than a ternary chain, and that is the fix: it read
|
|
44
|
+
* `period === "hour" ? "1 hour" : "1 day"`, so adding `week` and `month` to `ROLLUP_PERIODS` silently gave both
|
|
45
|
+
* a **one-day** window. A month bucket starting on the 1st then aggregated only the 1st, and every monthly figure
|
|
46
|
+
* was whatever happened on the first of the month — zero for most of any month, and never obviously wrong.
|
|
47
|
+
*
|
|
48
|
+
* Nothing failed: the migration's CHECK accepted the period, the bucket arithmetic truncated correctly, the row
|
|
49
|
+
* was written. Only the number was wrong. A total map makes the next period a compile error instead.
|
|
50
|
+
*/
|
|
51
|
+
const BUCKET_INTERVAL = {
|
|
52
|
+
hour: "1 hour",
|
|
53
|
+
day: "1 day",
|
|
54
|
+
week: "7 days",
|
|
55
|
+
// Postgres resolves `1 month` against the bucket's own start, so February gets 28 days and March 31 — the
|
|
56
|
+
// calendar behaviour the bucket boundaries already assume.
|
|
57
|
+
month: "1 month",
|
|
58
|
+
};
|
|
59
|
+
const intervalFor = (period) => BUCKET_INTERVAL[period];
|
|
60
|
+
export const createPostgresUsageRollupStore = (sql) => ({
|
|
61
|
+
/**
|
|
62
|
+
* Rebuild one bucket — the tenant's, or one principal's (#175).
|
|
63
|
+
*
|
|
64
|
+
* `principalId` absent aggregates every event in the bucket and writes the row where `principal_id IS NULL`;
|
|
65
|
+
* supplied aggregates that person's events and writes their row. The same statement either way, because the
|
|
66
|
+
* two are the same measurement at two grains and a second statement would be a second chance to disagree.
|
|
67
|
+
*
|
|
68
|
+
* `IS NOT DISTINCT FROM` in the conflict target rather than `=`, because the tenant row's `principal_id` is
|
|
69
|
+
* NULL and `NULL = NULL` is not true — an equality predicate would fail to match the existing row and try to
|
|
70
|
+
* insert a duplicate, which the partial unique index would then reject. The kind of bug that only appears on
|
|
71
|
+
* the *second* rebuild.
|
|
72
|
+
*/
|
|
73
|
+
async rebuild({ tenantId, period, bucketStart, principalId }) {
|
|
74
|
+
const rows = await sql.query(`WITH agg AS (
|
|
75
|
+
SELECT COALESCE(SUM(input_tokens), 0) AS input_tokens,
|
|
76
|
+
COALESCE(SUM(output_tokens), 0) AS output_tokens,
|
|
77
|
+
COALESCE(SUM(cached_input_tokens), 0) AS cached_input_tokens,
|
|
78
|
+
-- reasoning_tokens is nullable in usage_records (not every provider reports it), so the
|
|
79
|
+
-- COALESCE is inside the SUM as well as outside: SUM ignores NULLs, but a bucket where *every*
|
|
80
|
+
-- row is NULL sums to NULL rather than 0.
|
|
81
|
+
COALESCE(SUM(COALESCE(reasoning_tokens, 0)), 0) AS reasoning_tokens,
|
|
82
|
+
COALESCE(SUM(cost_minor_units), 0) AS cost_minor_units,
|
|
83
|
+
COUNT(*) AS event_count,
|
|
84
|
+
-- One currency per bucket. MIN rather than an arbitrary pick so the value is deterministic:
|
|
85
|
+
-- two rebuilds of the same bucket must agree, and a mixed-currency bucket is a configuration
|
|
86
|
+
-- problem the reconciliation report surfaces rather than something to paper over here.
|
|
87
|
+
COALESCE(MIN(currency), '') AS currency,
|
|
88
|
+
-- The high-water mark this rollup covers. Recorded so staleness is an integer comparison
|
|
89
|
+
-- rather than a clock comparison -- see 0019.
|
|
90
|
+
COALESCE(MAX(record_seq), 0) AS covers_seq
|
|
91
|
+
FROM usage_records
|
|
92
|
+
WHERE tenant_id = $1
|
|
93
|
+
AND occurred_at >= $3::timestamptz
|
|
94
|
+
AND occurred_at < $3::timestamptz + INTERVAL '${intervalFor(period)}'
|
|
95
|
+
-- The whole tenant when $4 is NULL, one person otherwise. One predicate rather than two queries.
|
|
96
|
+
AND ($4::text IS NULL OR principal_id = $4::text)
|
|
97
|
+
)
|
|
98
|
+
INSERT INTO usage_rollups (tenant_id, principal_id, period, bucket_start, input_tokens, output_tokens,
|
|
99
|
+
cached_input_tokens, reasoning_tokens, cost_minor_units, event_count,
|
|
100
|
+
currency, computed_at, covers_seq)
|
|
101
|
+
SELECT $1, $4::text, $2, $3::timestamptz, input_tokens, output_tokens, cached_input_tokens, reasoning_tokens,
|
|
102
|
+
-- clock_timestamp(), not now(). now() is the *transaction* timestamp and therefore constant
|
|
103
|
+
-- within one -- so a rebuild and an append in the same transaction would stamp the identical
|
|
104
|
+
-- instant, the staleness comparison would always hold, and the bucket would be rebuilt forever.
|
|
105
|
+
-- Found by the conformance suite, whose executor is transaction-scoped.
|
|
106
|
+
cost_minor_units, event_count, currency, clock_timestamp(), covers_seq
|
|
107
|
+
FROM agg
|
|
108
|
+
-- The upsert *is* the idempotency: a re-run replaces rather than accumulates, and two workers racing
|
|
109
|
+
-- this bucket write the same value. DO UPDATE rather than DO NOTHING because a rebuild after new events
|
|
110
|
+
-- must actually change the row.
|
|
111
|
+
-- The arbiter has to name the index that actually covers this row's grain — #175.
|
|
112
|
+
--
|
|
113
|
+
-- Uniqueness is two *partial* indexes, one where principal_id IS NULL and one where it is not, because a
|
|
114
|
+
-- NULL cannot participate in a normal unique constraint. A single ON CONFLICT clause therefore cannot
|
|
115
|
+
-- serve both: naming the tenant index while inserting a principal row finds no arbiter, inserts, and
|
|
116
|
+
-- violates the principal index on the *second* rebuild. Which is exactly what happened — the first
|
|
117
|
+
-- rebuild of each per-person bucket succeeded and every one after it threw a duplicate-key violation on
|
|
118
|
+
-- usage_rollups_principal_bucket_idx.
|
|
119
|
+
ON CONFLICT ${principalId === undefined
|
|
120
|
+
? "(tenant_id, period, bucket_start) WHERE principal_id IS NULL"
|
|
121
|
+
: "(tenant_id, principal_id, period, bucket_start) WHERE principal_id IS NOT NULL"} DO UPDATE SET
|
|
122
|
+
input_tokens = EXCLUDED.input_tokens,
|
|
123
|
+
output_tokens = EXCLUDED.output_tokens,
|
|
124
|
+
cached_input_tokens = EXCLUDED.cached_input_tokens,
|
|
125
|
+
reasoning_tokens = EXCLUDED.reasoning_tokens,
|
|
126
|
+
cost_minor_units = EXCLUDED.cost_minor_units,
|
|
127
|
+
event_count = EXCLUDED.event_count,
|
|
128
|
+
currency = EXCLUDED.currency,
|
|
129
|
+
computed_at = EXCLUDED.computed_at,
|
|
130
|
+
covers_seq = EXCLUDED.covers_seq
|
|
131
|
+
RETURNING ${COLUMNS}`, [tenantId, period, bucketStart, principalId ?? null]);
|
|
132
|
+
const row = rows[0];
|
|
133
|
+
if (row === undefined)
|
|
134
|
+
// Unreachable: the CTE always produces one row, even for an empty bucket. Returned rather than thrown so
|
|
135
|
+
// a caller never has to handle an impossible null.
|
|
136
|
+
return {
|
|
137
|
+
period,
|
|
138
|
+
bucketStart,
|
|
139
|
+
inputTokens: 0,
|
|
140
|
+
outputTokens: 0,
|
|
141
|
+
cachedInputTokens: 0,
|
|
142
|
+
reasoningTokens: 0,
|
|
143
|
+
costMinorUnits: 0,
|
|
144
|
+
eventCount: 0,
|
|
145
|
+
currency: "",
|
|
146
|
+
computedAt: new Date().toISOString(),
|
|
147
|
+
};
|
|
148
|
+
return toRollup(row);
|
|
149
|
+
},
|
|
150
|
+
async get({ tenantId, period, bucketStart, principalId }) {
|
|
151
|
+
const rows = await sql.query(`SELECT ${COLUMNS} FROM usage_rollups
|
|
152
|
+
WHERE tenant_id = $1 AND period = $2 AND bucket_start = $3::timestamptz
|
|
153
|
+
-- IS NOT DISTINCT FROM, so a NULL parameter matches the tenant row where = would match nothing.
|
|
154
|
+
AND principal_id IS NOT DISTINCT FROM $4::text`, [tenantId, period, bucketStart, principalId ?? null]);
|
|
155
|
+
// A foreign tenant's bucket yields no row, so no aggregate can span tenants and the answer is null without
|
|
156
|
+
// a comparison anyone could get wrong.
|
|
157
|
+
return rows[0] === undefined ? null : toRollup(rows[0]);
|
|
158
|
+
},
|
|
159
|
+
async list({ tenantId, period, from, to, limit, cursor, principalId }) {
|
|
160
|
+
const rows = await sql.query(`SELECT ${COLUMNS} FROM usage_rollups
|
|
161
|
+
WHERE tenant_id = $1 AND period = $2
|
|
162
|
+
-- The tenant's own buckets when $7 is NULL, one person's otherwise (#175). Without this predicate a
|
|
163
|
+
-- tenant chart would sum its own row *and* every principal's, double-counting everything.
|
|
164
|
+
AND principal_id IS NOT DISTINCT FROM $7::text
|
|
165
|
+
AND bucket_start >= $3::timestamptz
|
|
166
|
+
-- Exclusive upper bound, so adjacent ranges tile without a caller double-counting a boundary bucket.
|
|
167
|
+
AND bucket_start < $4::timestamptz
|
|
168
|
+
AND ($5::text IS NULL OR bucket_start > $5::timestamptz)
|
|
169
|
+
ORDER BY bucket_start
|
|
170
|
+
LIMIT $6`, [tenantId, period, from, to, cursor ?? null, limit + 1, principalId ?? null]);
|
|
171
|
+
const items = rows.slice(0, limit).map(toRollup);
|
|
172
|
+
const last = items[items.length - 1];
|
|
173
|
+
return rows.length > limit && last !== undefined
|
|
174
|
+
? { items, nextCursor: last.bucketStart }
|
|
175
|
+
: { items };
|
|
176
|
+
},
|
|
177
|
+
async sum({ tenantId, period, from, to, principalId }) {
|
|
178
|
+
const rows = await sql.query(
|
|
179
|
+
// Over the *rollups*, which is the point of AC-1: a spend query never scans raw records however much has
|
|
180
|
+
// been recorded.
|
|
181
|
+
`SELECT COALESCE(SUM(input_tokens), 0) AS input_tokens,
|
|
182
|
+
COALESCE(SUM(output_tokens), 0) AS output_tokens,
|
|
183
|
+
COALESCE(SUM(cached_input_tokens), 0) AS cached_input_tokens,
|
|
184
|
+
COALESCE(SUM(reasoning_tokens), 0) AS reasoning_tokens,
|
|
185
|
+
COALESCE(SUM(cost_minor_units), 0) AS cost_minor_units,
|
|
186
|
+
COALESCE(SUM(event_count), 0) AS event_count
|
|
187
|
+
FROM usage_rollups
|
|
188
|
+
WHERE tenant_id = $1 AND period = $2
|
|
189
|
+
-- Same predicate as list, and load-bearing for the same reason: summing across the principal
|
|
190
|
+
-- dimension would add each person's total to the tenant's and double every figure.
|
|
191
|
+
AND principal_id IS NOT DISTINCT FROM $5::text
|
|
192
|
+
AND bucket_start >= $3::timestamptz AND bucket_start < $4::timestamptz`, [tenantId, period, from, to, principalId ?? null]);
|
|
193
|
+
const r = rows[0];
|
|
194
|
+
return {
|
|
195
|
+
inputTokens: Number(r?.input_tokens ?? 0),
|
|
196
|
+
outputTokens: Number(r?.output_tokens ?? 0),
|
|
197
|
+
cachedInputTokens: Number(r?.cached_input_tokens ?? 0),
|
|
198
|
+
reasoningTokens: Number(r?.reasoning_tokens ?? 0),
|
|
199
|
+
costMinorUnits: Number(r?.cost_minor_units ?? 0),
|
|
200
|
+
eventCount: Number(r?.event_count ?? 0),
|
|
201
|
+
};
|
|
202
|
+
},
|
|
203
|
+
async listStaleBuckets({ tenantId, period, since, limit, cursor }) {
|
|
204
|
+
const rows = await sql.query(
|
|
205
|
+
// Derived from the ledger: a bucket is stale when it holds an event at or after its last computation, or
|
|
206
|
+
// has never been computed. So the job's work list needs no persisted cursor -- an interrupted run resumes
|
|
207
|
+
// by asking again, and there is nothing to lose.
|
|
208
|
+
`WITH buckets AS (
|
|
209
|
+
/**
|
|
210
|
+
* GROUPING SETS, so one pass finds both grains — #175.
|
|
211
|
+
*
|
|
212
|
+
* A per-principal rollup is only useful if something rebuilds it, and the job cannot enumerate which
|
|
213
|
+
* principals were active in a bucket without reading the ledger — which is the scan a rollup exists to
|
|
214
|
+
* avoid. This query already reads the ledger to decide staleness, so it is the one place that knows
|
|
215
|
+
* cheaply, and GROUPING SETS ((bucket), (bucket, principal)) gets both for the price of the pass it
|
|
216
|
+
* was already making.
|
|
217
|
+
*
|
|
218
|
+
* The (bucket) set produces the tenant row with a NULL principal_id, which is exactly how the tenant
|
|
219
|
+
* row is stored — so the LEFT JOIN below matches without a special case.
|
|
220
|
+
*/
|
|
221
|
+
SELECT date_trunc($2, occurred_at) AS bucket_start,
|
|
222
|
+
principal_id,
|
|
223
|
+
-- Which grouping set produced this row.
|
|
224
|
+
--
|
|
225
|
+
-- Necessary, not decoration: a NULL principal_id means "aggregated over every principal" in the
|
|
226
|
+
-- (bucket) set and "this event has no principal" in the (bucket, principal) set. Without
|
|
227
|
+
-- GROUPING they are indistinguishable, and a ledger of events with no principal produces the
|
|
228
|
+
-- same row twice — which is exactly what the conformance suite caught.
|
|
229
|
+
GROUPING(principal_id) AS is_tenant_row,
|
|
230
|
+
-- The newest *sequence* in this bucket, not the newest time. An event recorded late with an
|
|
231
|
+
-- occurred_at in the past still has a higher sequence, so its bucket is correctly stale.
|
|
232
|
+
MAX(record_seq) AS newest_seq
|
|
233
|
+
FROM usage_records
|
|
234
|
+
WHERE tenant_id = $1 AND occurred_at >= $3::timestamptz
|
|
235
|
+
GROUP BY GROUPING SETS ((1), (1, 2))
|
|
236
|
+
)
|
|
237
|
+
SELECT b.bucket_start, b.principal_id
|
|
238
|
+
FROM buckets b
|
|
239
|
+
LEFT JOIN usage_rollups r
|
|
240
|
+
ON r.tenant_id = $1 AND r.period = $2 AND r.bucket_start = b.bucket_start
|
|
241
|
+
AND r.principal_id IS NOT DISTINCT FROM b.principal_id
|
|
242
|
+
-- Parenthesised deliberately: AND binds tighter than OR, so without these the cursor filter would
|
|
243
|
+
-- apply only to the second branch and every never-computed bucket would come back on every page.
|
|
244
|
+
-- Strictly greater: an integer comparison with no ties, so a drained bucket is *not* listed.
|
|
245
|
+
WHERE (r.bucket_start IS NULL OR COALESCE(r.covers_seq, 0) < b.newest_seq)
|
|
246
|
+
AND ($4::text IS NULL OR b.bucket_start > $4::timestamptz)
|
|
247
|
+
-- Keep the tenant row, and a principal row only when there is an actual principal. Events with no
|
|
248
|
+
-- principal contribute to the tenant total and to no per-person total, which is what "unattributed"
|
|
249
|
+
-- honestly means.
|
|
250
|
+
AND (b.is_tenant_row = 1 OR b.principal_id IS NOT NULL)
|
|
251
|
+
-- The tenant row first within a bucket, so a job that stops mid-bucket has at least the total.
|
|
252
|
+
ORDER BY b.bucket_start, b.principal_id NULLS FIRST
|
|
253
|
+
LIMIT $5`, [tenantId, period, since, cursor ?? null, limit + 1]);
|
|
254
|
+
const items = rows.slice(0, limit).map((r) => ({
|
|
255
|
+
period,
|
|
256
|
+
bucketStart: iso(r.bucket_start),
|
|
257
|
+
...(r.principal_id === null || r.principal_id === undefined
|
|
258
|
+
? {}
|
|
259
|
+
: { principalId: r.principal_id }),
|
|
260
|
+
}));
|
|
261
|
+
const last = items[items.length - 1];
|
|
262
|
+
return rows.length > limit && last !== undefined
|
|
263
|
+
? { items, nextCursor: last.bucketStart }
|
|
264
|
+
: { items };
|
|
265
|
+
},
|
|
266
|
+
});
|
|
267
|
+
//# sourceMappingURL=rollups.js.map
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { ConversationRunCoordinator } from "../../persistence/index.js";
|
|
2
|
+
import type { SqlExecutor } from "./sql.js";
|
|
3
|
+
import type { TransactionRunner } from "./transaction.js";
|
|
4
|
+
export declare const createPostgresConversationRunCoordinator: (sql: SqlExecutor, runner: TransactionRunner) => ConversationRunCoordinator;
|
|
5
|
+
//# sourceMappingURL=run-coordinator.d.ts.map
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/** jsonb comes back parsed from PGlite and as text from some drivers; normalise both. */
|
|
2
|
+
const toQueue = (value) => {
|
|
3
|
+
const parsed = typeof value === "string" ? JSON.parse(value) : value;
|
|
4
|
+
return Array.isArray(parsed) ? parsed.map(String) : [];
|
|
5
|
+
};
|
|
6
|
+
const SELECT_SLOT_FOR_UPDATE = `SELECT active_run_id, queued
|
|
7
|
+
FROM conversation_run_slots
|
|
8
|
+
WHERE tenant_id = $1 AND conversation_id = $2
|
|
9
|
+
FOR UPDATE`;
|
|
10
|
+
const WRITE_SLOT = `UPDATE conversation_run_slots
|
|
11
|
+
SET active_run_id = $3, queued = $4::jsonb, updated_at = now()
|
|
12
|
+
WHERE tenant_id = $1 AND conversation_id = $2`;
|
|
13
|
+
export const createPostgresConversationRunCoordinator = (sql, runner) => {
|
|
14
|
+
/**
|
|
15
|
+
* Whether the slot's current holder can be displaced.
|
|
16
|
+
*
|
|
17
|
+
* Liveness comes from the run's own lease (#93) rather than a second clock on the slot. Two leases
|
|
18
|
+
* for one fact drift, and when they disagree there is no principled tiebreak.
|
|
19
|
+
*
|
|
20
|
+
* The `rows[0]?.dead === true` shape carries the load-bearing half: **no run row means unknown, not
|
|
21
|
+
* dead.** `claimOrEnqueue` is legitimately called before the run row is committed, so treating a
|
|
22
|
+
* missing row as dead would let a fresh claim steal a live run's slot — a correctness bug much
|
|
23
|
+
* worse than recovering a crashed worker's slot slowly.
|
|
24
|
+
*/
|
|
25
|
+
const holderIsDead = async (tx, tenantId, activeRunId) => {
|
|
26
|
+
const rows = await tx.query(`SELECT (
|
|
27
|
+
status IN ('completed', 'failed', 'cancelled')
|
|
28
|
+
OR (status = 'running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= now())
|
|
29
|
+
) AS dead
|
|
30
|
+
FROM runs
|
|
31
|
+
WHERE tenant_id = $1 AND id = $2`, [tenantId, activeRunId]);
|
|
32
|
+
return rows[0]?.dead === true;
|
|
33
|
+
};
|
|
34
|
+
return {
|
|
35
|
+
async claimOrEnqueue({ tenantId, conversationId, runId }) {
|
|
36
|
+
return runner.transaction(async (tx) => {
|
|
37
|
+
// An idle conversation still needs a row to lock. Separate from the lock because
|
|
38
|
+
// `INSERT … ON CONFLICT DO NOTHING` does not lock the conflicting row it declined to touch.
|
|
39
|
+
await tx.query(`INSERT INTO conversation_run_slots (tenant_id, conversation_id, queued, updated_at)
|
|
40
|
+
VALUES ($1, $2, '[]'::jsonb, now())
|
|
41
|
+
ON CONFLICT (tenant_id, conversation_id) DO NOTHING`, [tenantId, conversationId]);
|
|
42
|
+
const rows = await tx.query(SELECT_SLOT_FOR_UPDATE, [tenantId, conversationId]);
|
|
43
|
+
const row = rows[0];
|
|
44
|
+
// Unreachable in practice: the insert above guarantees a row, and the lock is held. Kept
|
|
45
|
+
// because returning a wrong "started" here would break single-flight silently.
|
|
46
|
+
if (!row)
|
|
47
|
+
throw new Error(`conversation_run_slots row missing for ${conversationId}`);
|
|
48
|
+
const queue = toQueue(row.queued);
|
|
49
|
+
const active = row.active_run_id;
|
|
50
|
+
const free = active === null || active === runId || (await holderIsDead(tx, tenantId, active));
|
|
51
|
+
if (free) {
|
|
52
|
+
// Claiming also drops the run from the backlog: a queued run that gets promoted by its own
|
|
53
|
+
// retry must not remain listed behind itself.
|
|
54
|
+
const remaining = queue.filter((r) => r !== runId);
|
|
55
|
+
await tx.query(WRITE_SLOT, [tenantId, conversationId, runId, JSON.stringify(remaining)]);
|
|
56
|
+
return { status: "started", position: 0 };
|
|
57
|
+
}
|
|
58
|
+
const next = queue.includes(runId) ? queue : [...queue, runId];
|
|
59
|
+
if (next !== queue) {
|
|
60
|
+
await tx.query(WRITE_SLOT, [tenantId, conversationId, active, JSON.stringify(next)]);
|
|
61
|
+
}
|
|
62
|
+
return { status: "queued", position: next.indexOf(runId) + 1 };
|
|
63
|
+
});
|
|
64
|
+
},
|
|
65
|
+
async releaseAndPromote({ tenantId, conversationId, runId }) {
|
|
66
|
+
return runner.transaction(async (tx) => {
|
|
67
|
+
const rows = await tx.query(SELECT_SLOT_FOR_UPDATE, [tenantId, conversationId]);
|
|
68
|
+
const row = rows[0];
|
|
69
|
+
if (!row)
|
|
70
|
+
return null;
|
|
71
|
+
// A release from a run that does not hold the slot changes nothing — it must not unseat the
|
|
72
|
+
// holder and must not promote anyone. Matches the reference adapter.
|
|
73
|
+
if (row.active_run_id !== runId)
|
|
74
|
+
return null;
|
|
75
|
+
const queue = toQueue(row.queued);
|
|
76
|
+
const promoted = queue.shift() ?? null;
|
|
77
|
+
// Release and promote in the same statement, inside the same lock: the port's "no
|
|
78
|
+
// release→dequeue→claim gap" requirement is exactly the window this closes.
|
|
79
|
+
await tx.query(WRITE_SLOT, [tenantId, conversationId, promoted, JSON.stringify(queue)]);
|
|
80
|
+
return promoted;
|
|
81
|
+
});
|
|
82
|
+
},
|
|
83
|
+
async active({ tenantId, conversationId }) {
|
|
84
|
+
const rows = await sql.query(`SELECT active_run_id FROM conversation_run_slots
|
|
85
|
+
WHERE tenant_id = $1 AND conversation_id = $2`, [tenantId, conversationId]);
|
|
86
|
+
return (rows[0]?.active_run_id ?? null);
|
|
87
|
+
},
|
|
88
|
+
async depth({ tenantId, conversationId }) {
|
|
89
|
+
// Counted in the database rather than by fetching the array: depth is asked for far more often
|
|
90
|
+
// than the queue's contents are needed.
|
|
91
|
+
const rows = await sql.query(`SELECT jsonb_array_length(queued) AS depth FROM conversation_run_slots
|
|
92
|
+
WHERE tenant_id = $1 AND conversation_id = $2`, [tenantId, conversationId]);
|
|
93
|
+
const raw = rows[0]?.depth;
|
|
94
|
+
return raw === undefined ? 0 : Number(raw);
|
|
95
|
+
},
|
|
96
|
+
};
|
|
97
|
+
};
|
|
98
|
+
//# sourceMappingURL=run-coordinator.js.map
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PostgreSQL `RunEventLog` (#94). The durable half of streaming: catch-up after a reconnect reads
|
|
3
|
+
* this log, and worker crash-recovery reconciles against it (the C1 fix from the REQ-005 review).
|
|
4
|
+
*
|
|
5
|
+
* Two decisions carry the design:
|
|
6
|
+
*
|
|
7
|
+
* 1. **The caller's `sequence` is stored verbatim.** It is the worker's checkpoint cursor and the
|
|
8
|
+
* client's reconnect cursor, so the store must never renumber. The SPEC originally called for
|
|
9
|
+
* server-side allocation (`MAX(sequence)+1`), which would have overwritten both.
|
|
10
|
+
* 2. **`ON CONFLICT DO NOTHING`.** A duplicate sequence is a silent no-op, matching the reference
|
|
11
|
+
* adapter — a recovered worker re-emits events, so a retried append is normal rather than
|
|
12
|
+
* exceptional. The composite primary key makes the collision impossible; this turns it into the
|
|
13
|
+
* idempotent no-op the port promises. The first write wins, so history never mutates under a
|
|
14
|
+
* client that already read it.
|
|
15
|
+
*
|
|
16
|
+
* `append` takes whatever executor it is handed, so a transaction-scoped executor puts the write
|
|
17
|
+
* inside that transaction — which is what lets `emit()` keep writing the event *before* the
|
|
18
|
+
* checkpoint, atomically, without this store knowing about transactions at all.
|
|
19
|
+
*/
|
|
20
|
+
import type { RunEventLog } from "../../core/events.js";
|
|
21
|
+
import type { RunId, TenantId } from "../../core/ids.js";
|
|
22
|
+
import type { SqlExecutor } from "./sql.js";
|
|
23
|
+
export declare const createPostgresRunEventLog: (sql: SqlExecutor) => RunEventLog;
|
|
24
|
+
/** Re-exported for tests that need the row shape without importing the module's internals. */
|
|
25
|
+
export type { RunId, TenantId };
|
|
26
|
+
//# sourceMappingURL=run-event-log.d.ts.map
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { parseRunEvent } from "../../core/validation.js";
|
|
2
|
+
/**
|
|
3
|
+
* Validate on read rather than trusting the column. A corrupt or hand-edited row otherwise flows
|
|
4
|
+
* straight into a client's event stream, where the failure surfaces far from its cause.
|
|
5
|
+
*/
|
|
6
|
+
const toEvent = (row) => parseRunEvent(typeof row.event === "string" ? JSON.parse(row.event) : row.event);
|
|
7
|
+
export const createPostgresRunEventLog = (sql) => ({
|
|
8
|
+
async append({ tenantId, event }) {
|
|
9
|
+
await sql.query(`INSERT INTO run_events (tenant_id, run_id, sequence, type, event)
|
|
10
|
+
VALUES ($1, $2, $3, $4, $5::jsonb)
|
|
11
|
+
ON CONFLICT (tenant_id, run_id, sequence) DO NOTHING`, [tenantId, event.runId, event.sequence, event.type, JSON.stringify(event)]);
|
|
12
|
+
},
|
|
13
|
+
async listAfter({ tenantId, runId, after, limit }) {
|
|
14
|
+
// `sequence > $3 ORDER BY sequence` is a forward range scan on the primary key — the whole point
|
|
15
|
+
// of keying on (tenant_id, run_id, sequence) rather than a surrogate id.
|
|
16
|
+
const rows = await sql.query(`SELECT event FROM run_events
|
|
17
|
+
WHERE tenant_id = $1 AND run_id = $2 AND sequence > $3
|
|
18
|
+
ORDER BY sequence
|
|
19
|
+
${limit === undefined ? "" : "LIMIT $4"}`, limit === undefined ? [tenantId, runId, after] : [tenantId, runId, after, limit]);
|
|
20
|
+
return rows.map(toEvent);
|
|
21
|
+
},
|
|
22
|
+
async latestSequence({ tenantId, runId }) {
|
|
23
|
+
// COALESCE, not an empty-result branch: a run with no events has head 0, which is the cursor a
|
|
24
|
+
// fresh client sends. Recovery compares against this to decide what the checkpoint missed.
|
|
25
|
+
const rows = await sql.query(`SELECT COALESCE(MAX(sequence), 0) AS head FROM run_events WHERE tenant_id = $1 AND run_id = $2`, [tenantId, runId]);
|
|
26
|
+
const head = rows[0]?.head ?? 0;
|
|
27
|
+
return typeof head === "number" ? head : Number(head);
|
|
28
|
+
},
|
|
29
|
+
});
|
|
30
|
+
//# sourceMappingURL=run-event-log.js.map
|