@retinue/agentkit 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/README.md +310 -0
- package/dist/adapters/bullmq/consumer.d.ts +33 -0
- package/dist/adapters/bullmq/consumer.js +41 -0
- package/dist/adapters/bullmq/dispatcher.d.ts +74 -0
- package/dist/adapters/bullmq/dispatcher.js +160 -0
- package/dist/adapters/bullmq/export.d.ts +31 -0
- package/dist/adapters/bullmq/export.js +53 -0
- package/dist/adapters/bullmq/extraction.d.ts +42 -0
- package/dist/adapters/bullmq/extraction.js +63 -0
- package/dist/adapters/bullmq/index.d.ts +13 -0
- package/dist/adapters/bullmq/index.js +13 -0
- package/dist/adapters/bullmq/lock.d.ts +77 -0
- package/dist/adapters/bullmq/lock.js +126 -0
- package/dist/adapters/bullmq/queue.d.ts +50 -0
- package/dist/adapters/bullmq/queue.js +81 -0
- package/dist/adapters/memory/artifact-exports.d.ts +11 -0
- package/dist/adapters/memory/artifact-exports.js +102 -0
- package/dist/adapters/memory/artifacts.d.ts +15 -0
- package/dist/adapters/memory/artifacts.js +134 -0
- package/dist/adapters/memory/blobs.d.ts +7 -0
- package/dist/adapters/memory/blobs.js +27 -0
- package/dist/adapters/memory/evaluation.d.ts +18 -0
- package/dist/adapters/memory/evaluation.js +148 -0
- package/dist/adapters/memory/files.d.ts +27 -0
- package/dist/adapters/memory/files.js +0 -0
- package/dist/adapters/memory/flows.d.ts +16 -0
- package/dist/adapters/memory/flows.js +117 -0
- package/dist/adapters/memory/hitl.d.ts +9 -0
- package/dist/adapters/memory/hitl.js +130 -0
- package/dist/adapters/memory/idempotency.d.ts +13 -0
- package/dist/adapters/memory/idempotency.js +32 -0
- package/dist/adapters/memory/index.d.ts +39 -0
- package/dist/adapters/memory/index.js +107 -0
- package/dist/adapters/memory/knowledge.d.ts +43 -0
- package/dist/adapters/memory/knowledge.js +248 -0
- package/dist/adapters/memory/mcp.d.ts +9 -0
- package/dist/adapters/memory/mcp.js +37 -0
- package/dist/adapters/memory/message-store.d.ts +17 -0
- package/dist/adapters/memory/message-store.js +70 -0
- package/dist/adapters/memory/principal-memory.d.ts +7 -0
- package/dist/adapters/memory/principal-memory.js +83 -0
- package/dist/adapters/memory/runtime.d.ts +29 -0
- package/dist/adapters/memory/runtime.js +0 -0
- package/dist/adapters/memory/sessions.d.ts +29 -0
- package/dist/adapters/memory/sessions.js +0 -0
- package/dist/adapters/memory/skills.d.ts +10 -0
- package/dist/adapters/memory/skills.js +41 -0
- package/dist/adapters/memory/thread-summary.d.ts +7 -0
- package/dist/adapters/memory/thread-summary.js +29 -0
- package/dist/adapters/memory/usage-limits.d.ts +13 -0
- package/dist/adapters/memory/usage-limits.js +72 -0
- package/dist/adapters/memory/usage.d.ts +16 -0
- package/dist/adapters/memory/usage.js +279 -0
- package/dist/adapters/otel/index.d.ts +111 -0
- package/dist/adapters/otel/index.js +133 -0
- package/dist/adapters/postgres/artifact-exports.d.ts +12 -0
- package/dist/adapters/postgres/artifact-exports.js +117 -0
- package/dist/adapters/postgres/artifacts.d.ts +16 -0
- package/dist/adapters/postgres/artifacts.js +172 -0
- package/dist/adapters/postgres/checkpoint-store.d.ts +16 -0
- package/dist/adapters/postgres/checkpoint-store.js +34 -0
- package/dist/adapters/postgres/config.d.ts +15 -0
- package/dist/adapters/postgres/config.js +187 -0
- package/dist/adapters/postgres/conversation-store.d.ts +4 -0
- package/dist/adapters/postgres/conversation-store.js +82 -0
- package/dist/adapters/postgres/evaluation.d.ts +17 -0
- package/dist/adapters/postgres/evaluation.js +193 -0
- package/dist/adapters/postgres/file-content.d.ts +30 -0
- package/dist/adapters/postgres/file-content.js +111 -0
- package/dist/adapters/postgres/files.d.ts +19 -0
- package/dist/adapters/postgres/files.js +209 -0
- package/dist/adapters/postgres/flows.d.ts +20 -0
- package/dist/adapters/postgres/flows.js +206 -0
- package/dist/adapters/postgres/hitl.d.ts +5 -0
- package/dist/adapters/postgres/hitl.js +247 -0
- package/dist/adapters/postgres/index.d.ts +35 -0
- package/dist/adapters/postgres/index.js +35 -0
- package/dist/adapters/postgres/knowledge.d.ts +48 -0
- package/dist/adapters/postgres/knowledge.js +255 -0
- package/dist/adapters/postgres/memory.d.ts +14 -0
- package/dist/adapters/postgres/memory.js +194 -0
- package/dist/adapters/postgres/message-store.d.ts +11 -0
- package/dist/adapters/postgres/message-store.js +145 -0
- package/dist/adapters/postgres/migrations.d.ts +69 -0
- package/dist/adapters/postgres/migrations.js +1594 -0
- package/dist/adapters/postgres/pg-executor.d.ts +19 -0
- package/dist/adapters/postgres/pg-executor.js +32 -0
- package/dist/adapters/postgres/retention.d.ts +26 -0
- package/dist/adapters/postgres/retention.js +59 -0
- package/dist/adapters/postgres/rollups.d.ts +17 -0
- package/dist/adapters/postgres/rollups.js +267 -0
- package/dist/adapters/postgres/run-coordinator.d.ts +5 -0
- package/dist/adapters/postgres/run-coordinator.js +98 -0
- package/dist/adapters/postgres/run-event-log.d.ts +26 -0
- package/dist/adapters/postgres/run-event-log.js +30 -0
- package/dist/adapters/postgres/run-store.d.ts +4 -0
- package/dist/adapters/postgres/run-store.js +199 -0
- package/dist/adapters/postgres/schema.d.ts +39 -0
- package/dist/adapters/postgres/schema.js +70 -0
- package/dist/adapters/postgres/session-state.d.ts +7 -0
- package/dist/adapters/postgres/session-state.js +99 -0
- package/dist/adapters/postgres/sql.d.ts +8 -0
- package/dist/adapters/postgres/sql.js +2 -0
- package/dist/adapters/postgres/transaction.d.ts +37 -0
- package/dist/adapters/postgres/transaction.js +93 -0
- package/dist/adapters/postgres/unit-of-work.d.ts +18 -0
- package/dist/adapters/postgres/unit-of-work.js +8 -0
- package/dist/adapters/postgres/usage-limits.d.ts +15 -0
- package/dist/adapters/postgres/usage-limits.js +136 -0
- package/dist/adapters/postgres/usage.d.ts +15 -0
- package/dist/adapters/postgres/usage.js +226 -0
- package/dist/adapters/redis/index.d.ts +9 -0
- package/dist/adapters/redis/index.js +9 -0
- package/dist/adapters/redis/realtime.d.ts +74 -0
- package/dist/adapters/redis/realtime.js +112 -0
- package/dist/adapters/supabase/index.d.ts +88 -0
- package/dist/adapters/supabase/index.js +84 -0
- package/dist/adapters/supabase/realtime.d.ts +30 -0
- package/dist/adapters/supabase/realtime.js +53 -0
- package/dist/adapters/supabase/rls.d.ts +99 -0
- package/dist/adapters/supabase/rls.js +216 -0
- package/dist/adapters/supabase/storage.d.ts +50 -0
- package/dist/adapters/supabase/storage.js +207 -0
- package/dist/agents/agent.d.ts +66 -0
- package/dist/agents/agent.js +209 -0
- package/dist/agents/define.d.ts +21 -0
- package/dist/agents/define.js +63 -0
- package/dist/agents/engine.d.ts +98 -0
- package/dist/agents/engine.js +462 -0
- package/dist/agents/index.d.ts +50 -0
- package/dist/agents/index.js +17 -0
- package/dist/artifacts/index.d.ts +114 -0
- package/dist/artifacts/index.js +219 -0
- package/dist/authorization/index.d.ts +76 -0
- package/dist/authorization/index.js +76 -0
- package/dist/capabilities/index.d.ts +120 -0
- package/dist/capabilities/index.js +167 -0
- package/dist/capabilities/runtime.d.ts +89 -0
- package/dist/capabilities/runtime.js +84 -0
- package/dist/citations/index.d.ts +161 -0
- package/dist/citations/index.js +182 -0
- package/dist/context/assembler.d.ts +82 -0
- package/dist/context/assembler.js +129 -0
- package/dist/context/compaction.d.ts +45 -0
- package/dist/context/compaction.js +55 -0
- package/dist/context/index.d.ts +75 -0
- package/dist/context/index.js +17 -0
- package/dist/core/content-parts.d.ts +194 -0
- package/dist/core/content-parts.js +23 -0
- package/dist/core/context.d.ts +51 -0
- package/dist/core/context.js +9 -0
- package/dist/core/env.d.ts +25 -0
- package/dist/core/env.js +41 -0
- package/dist/core/errors.d.ts +30 -0
- package/dist/core/errors.js +65 -0
- package/dist/core/events.d.ts +139 -0
- package/dist/core/events.js +99 -0
- package/dist/core/ids.d.ts +52 -0
- package/dist/core/ids.js +13 -0
- package/dist/core/index.d.ts +9 -0
- package/dist/core/index.js +9 -0
- package/dist/core/tokens.d.ts +22 -0
- package/dist/core/tokens.js +22 -0
- package/dist/core/validation.d.ts +34 -0
- package/dist/core/validation.js +176 -0
- package/dist/documents/extraction.d.ts +121 -0
- package/dist/documents/extraction.js +293 -0
- package/dist/documents/index.d.ts +199 -0
- package/dist/documents/index.js +65 -0
- package/dist/documents/parsers/pdf.d.ts +47 -0
- package/dist/documents/parsers/pdf.js +508 -0
- package/dist/documents/parsers/text.d.ts +59 -0
- package/dist/documents/parsers/text.js +325 -0
- package/dist/documents/read-tool.d.ts +52 -0
- package/dist/documents/read-tool.js +109 -0
- package/dist/documents/render.d.ts +29 -0
- package/dist/documents/render.js +59 -0
- package/dist/documents/vision.d.ts +159 -0
- package/dist/documents/vision.js +214 -0
- package/dist/entries/adapters-bullmq.d.ts +8 -0
- package/dist/entries/adapters-bullmq.js +8 -0
- package/dist/entries/adapters-otel.d.ts +13 -0
- package/dist/entries/adapters-otel.js +13 -0
- package/dist/entries/adapters-postgres.d.ts +10 -0
- package/dist/entries/adapters-postgres.js +10 -0
- package/dist/entries/adapters-redis.d.ts +3 -0
- package/dist/entries/adapters-redis.js +3 -0
- package/dist/entries/context.d.ts +20 -0
- package/dist/entries/context.js +20 -0
- package/dist/entries/flows.d.ts +15 -0
- package/dist/entries/flows.js +15 -0
- package/dist/entries/hitl.d.ts +10 -0
- package/dist/entries/hitl.js +10 -0
- package/dist/entries/knowledge.d.ts +18 -0
- package/dist/entries/knowledge.js +19 -0
- package/dist/entries/mcp.d.ts +10 -0
- package/dist/entries/mcp.js +10 -0
- package/dist/entries/observability.d.ts +14 -0
- package/dist/entries/observability.js +16 -0
- package/dist/entries/persistence.d.ts +11 -0
- package/dist/entries/persistence.js +11 -0
- package/dist/entries/providers.d.ts +14 -0
- package/dist/entries/providers.js +14 -0
- package/dist/entries/runtime.d.ts +13 -0
- package/dist/entries/runtime.js +13 -0
- package/dist/entries/server.d.ts +24 -0
- package/dist/entries/server.js +24 -0
- package/dist/entries/tools.d.ts +21 -0
- package/dist/entries/tools.js +21 -0
- package/dist/entries/usage.d.ts +10 -0
- package/dist/entries/usage.js +10 -0
- package/dist/evaluation/gate.d.ts +168 -0
- package/dist/evaluation/gate.js +180 -0
- package/dist/evaluation/graders.d.ts +125 -0
- package/dist/evaluation/graders.js +203 -0
- package/dist/evaluation/index.d.ts +120 -0
- package/dist/evaluation/index.js +183 -0
- package/dist/evaluation/judge.d.ts +75 -0
- package/dist/evaluation/judge.js +111 -0
- package/dist/export/index.d.ts +162 -0
- package/dist/export/index.js +363 -0
- package/dist/export/markdown.d.ts +19 -0
- package/dist/export/markdown.js +29 -0
- package/dist/export/pdf.d.ts +73 -0
- package/dist/export/pdf.js +407 -0
- package/dist/files/context.d.ts +97 -0
- package/dist/files/context.js +185 -0
- package/dist/files/index.d.ts +210 -0
- package/dist/files/index.js +338 -0
- package/dist/files/read-tool.d.ts +81 -0
- package/dist/files/read-tool.js +163 -0
- package/dist/files/turn-parts.d.ts +96 -0
- package/dist/files/turn-parts.js +171 -0
- package/dist/flows/index.d.ts +270 -0
- package/dist/flows/index.js +62 -0
- package/dist/flows/interpreter.d.ts +146 -0
- package/dist/flows/interpreter.js +426 -0
- package/dist/flows/runner.d.ts +145 -0
- package/dist/flows/runner.js +270 -0
- package/dist/graphql/index.d.ts +8 -0
- package/dist/graphql/index.js +8 -0
- package/dist/graphql/resolvers.d.ts +237 -0
- package/dist/graphql/resolvers.js +253 -0
- package/dist/graphql/schema.d.ts +11 -0
- package/dist/graphql/schema.js +258 -0
- package/dist/graphql/sse.d.ts +77 -0
- package/dist/graphql/sse.js +100 -0
- package/dist/hitl/approved-execution.d.ts +127 -0
- package/dist/hitl/approved-execution.js +177 -0
- package/dist/hitl/index.d.ts +79 -0
- package/dist/hitl/index.js +12 -0
- package/dist/hitl/service.d.ts +221 -0
- package/dist/hitl/service.js +268 -0
- package/dist/idempotency/index.d.ts +70 -0
- package/dist/idempotency/index.js +59 -0
- package/dist/index.d.ts +103 -0
- package/dist/index.js +60 -0
- package/dist/knowledge/chunking.d.ts +57 -0
- package/dist/knowledge/chunking.js +158 -0
- package/dist/knowledge/index.d.ts +119 -0
- package/dist/knowledge/index.js +166 -0
- package/dist/knowledge/retrieval.d.ts +146 -0
- package/dist/knowledge/retrieval.js +170 -0
- package/dist/loadtest/harness.d.ts +168 -0
- package/dist/loadtest/harness.js +507 -0
- package/dist/loadtest/index.d.ts +13 -0
- package/dist/loadtest/index.js +13 -0
- package/dist/loadtest/injection.d.ts +89 -0
- package/dist/loadtest/injection.js +147 -0
- package/dist/loadtest/metrics.d.ts +197 -0
- package/dist/loadtest/metrics.js +160 -0
- package/dist/loadtest/runbooks.d.ts +28 -0
- package/dist/loadtest/runbooks.js +159 -0
- package/dist/loadtest/scenario.d.ts +104 -0
- package/dist/loadtest/scenario.js +208 -0
- package/dist/mcp/egress.d.ts +53 -0
- package/dist/mcp/egress.js +115 -0
- package/dist/mcp/index.d.ts +93 -0
- package/dist/mcp/index.js +33 -0
- package/dist/mcp/provider.d.ts +62 -0
- package/dist/mcp/provider.js +0 -0
- package/dist/models/index.d.ts +98 -0
- package/dist/models/index.js +74 -0
- package/dist/models/pricing.d.ts +24 -0
- package/dist/models/pricing.js +37 -0
- package/dist/models/provider-factory.d.ts +31 -0
- package/dist/models/provider-factory.js +67 -0
- package/dist/models/streaming.d.ts +145 -0
- package/dist/models/streaming.js +272 -0
- package/dist/models/vision.d.ts +38 -0
- package/dist/models/vision.js +62 -0
- package/dist/persistence/index.d.ts +1654 -0
- package/dist/persistence/index.js +226 -0
- package/dist/principal-memory/index.d.ts +106 -0
- package/dist/principal-memory/index.js +89 -0
- package/dist/retention/index.d.ts +89 -0
- package/dist/retention/index.js +70 -0
- package/dist/runtime/checkpoint.d.ts +37 -0
- package/dist/runtime/checkpoint.js +22 -0
- package/dist/runtime/index.d.ts +118 -0
- package/dist/runtime/index.js +69 -0
- package/dist/runtime/retry.d.ts +95 -0
- package/dist/runtime/retry.js +126 -0
- package/dist/runtime/serialization.d.ts +85 -0
- package/dist/runtime/serialization.js +95 -0
- package/dist/runtime/streaming.d.ts +54 -0
- package/dist/runtime/streaming.js +115 -0
- package/dist/runtime/worker.d.ts +130 -0
- package/dist/runtime/worker.js +405 -0
- package/dist/security/checklist.d.ts +53 -0
- package/dist/security/checklist.js +204 -0
- package/dist/security/findings.d.ts +56 -0
- package/dist/security/findings.js +168 -0
- package/dist/security/index.d.ts +14 -0
- package/dist/security/index.js +14 -0
- package/dist/security/prompt-safety.d.ts +100 -0
- package/dist/security/prompt-safety.js +133 -0
- package/dist/server/boot.d.ts +32 -0
- package/dist/server/boot.js +36 -0
- package/dist/server/cli-worker.d.ts +37 -0
- package/dist/server/cli-worker.js +151 -0
- package/dist/server/cli.d.ts +27 -0
- package/dist/server/cli.js +74 -0
- package/dist/server/config.d.ts +42 -0
- package/dist/server/config.js +127 -0
- package/dist/server/health.d.ts +59 -0
- package/dist/server/health.js +90 -0
- package/dist/server/host.d.ts +39 -0
- package/dist/server/host.js +124 -0
- package/dist/server/index.d.ts +15 -0
- package/dist/server/index.js +15 -0
- package/dist/server/main.d.ts +16 -0
- package/dist/server/main.js +31 -0
- package/dist/server/sse-route.d.ts +21 -0
- package/dist/server/sse-route.js +282 -0
- package/dist/skills/index.d.ts +67 -0
- package/dist/skills/index.js +31 -0
- package/dist/skills/resolver.d.ts +54 -0
- package/dist/skills/resolver.js +121 -0
- package/dist/teams/index.d.ts +93 -0
- package/dist/teams/index.js +207 -0
- package/dist/telemetry/index.d.ts +157 -0
- package/dist/telemetry/index.js +71 -0
- package/dist/telemetry/instrument.d.ts +108 -0
- package/dist/telemetry/instrument.js +232 -0
- package/dist/telemetry/log-events.d.ts +17 -0
- package/dist/telemetry/log-events.js +58 -0
- package/dist/telemetry/metrics.d.ts +123 -0
- package/dist/telemetry/metrics.js +135 -0
- package/dist/telemetry/noop.d.ts +39 -0
- package/dist/telemetry/noop.js +143 -0
- package/dist/telemetry/redaction.d.ts +64 -0
- package/dist/telemetry/redaction.js +153 -0
- package/dist/telemetry/spans.d.ts +56 -0
- package/dist/telemetry/spans.js +78 -0
- package/dist/telemetry/trace-context.d.ts +55 -0
- package/dist/telemetry/trace-context.js +60 -0
- package/dist/toolkit/compute.d.ts +53 -0
- package/dist/toolkit/compute.js +152 -0
- package/dist/toolkit/data.d.ts +98 -0
- package/dist/toolkit/data.js +235 -0
- package/dist/toolkit/http.d.ts +113 -0
- package/dist/toolkit/http.js +205 -0
- package/dist/toolkit/index.d.ts +21 -0
- package/dist/toolkit/index.js +17 -0
- package/dist/toolkit/web.d.ts +107 -0
- package/dist/toolkit/web.js +147 -0
- package/dist/tools/define.d.ts +25 -0
- package/dist/tools/define.js +45 -0
- package/dist/tools/delegating.d.ts +132 -0
- package/dist/tools/delegating.js +211 -0
- package/dist/tools/index.d.ts +129 -0
- package/dist/tools/index.js +33 -0
- package/dist/tools/library/compute.d.ts +11 -0
- package/dist/tools/library/compute.js +46 -0
- package/dist/tools/library/data.d.ts +16 -0
- package/dist/tools/library/data.js +92 -0
- package/dist/tools/library/http.d.ts +28 -0
- package/dist/tools/library/http.js +71 -0
- package/dist/tools/library/index.d.ts +97 -0
- package/dist/tools/library/index.js +134 -0
- package/dist/tools/library/knowledge.d.ts +39 -0
- package/dist/tools/library/knowledge.js +58 -0
- package/dist/tools/library/web.d.ts +19 -0
- package/dist/tools/library/web.js +65 -0
- package/dist/tools/meta-tools.d.ts +19 -0
- package/dist/tools/meta-tools.js +36 -0
- package/dist/tools/registry.d.ts +146 -0
- package/dist/tools/registry.js +291 -0
- package/dist/usage/index.d.ts +105 -0
- package/dist/usage/index.js +20 -0
- package/dist/usage/quota.d.ts +258 -0
- package/dist/usage/quota.js +510 -0
- package/dist/usage/recorder.d.ts +29 -0
- package/dist/usage/recorder.js +96 -0
- package/dist/usage/rollups.d.ts +121 -0
- package/dist/usage/rollups.js +157 -0
- package/dist/worker/export.d.ts +57 -0
- package/dist/worker/export.js +81 -0
- package/dist/worker/extraction.d.ts +57 -0
- package/dist/worker/extraction.js +84 -0
- package/dist/worker/main.d.ts +103 -0
- package/dist/worker/main.js +159 -0
- package/package.json +187 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BullMQ `ExportDispatcher` (#134).
|
|
3
|
+
*
|
|
4
|
+
* A third queue, and for the same reason extraction got its own: a hundred-page PDF render must not sit in
|
|
5
|
+
* front of a user's next message, and rendering is CPU-bound where a run waits on a provider. The id, escaping
|
|
6
|
+
* and timeout behaviour are `dispatcher.ts`'s, reused rather than re-derived — the length-prefix fix exists
|
|
7
|
+
* because `${tenant}-${id}` is *ambiguous*, and a second implementation is a second chance to forget that.
|
|
8
|
+
*/
|
|
9
|
+
import type { ExportDispatcher } from "../../export/index.js";
|
|
10
|
+
import { type JobDispatcherOptions } from "./dispatcher.js";
|
|
11
|
+
/** Hyphen, not a colon: BullMQ rejects a queue name containing `:` outright. See `RUN_QUEUE_NAME`. */
|
|
12
|
+
export declare const EXPORT_QUEUE_NAME = "agentkit-exports";
|
|
13
|
+
export declare const EXPORT_JOB_NAME = "export";
|
|
14
|
+
export type ExportJobData = {
|
|
15
|
+
readonly tenantId: string;
|
|
16
|
+
readonly exportId: string;
|
|
17
|
+
};
|
|
18
|
+
export interface ExportQueue {
|
|
19
|
+
add(name: string, data: ExportJobData, opts?: {
|
|
20
|
+
readonly jobId?: string;
|
|
21
|
+
readonly attempts?: number;
|
|
22
|
+
}): Promise<unknown>;
|
|
23
|
+
close?(): Promise<void>;
|
|
24
|
+
}
|
|
25
|
+
/** Tenant-qualified and unambiguous, built by the same function a run's id is. */
|
|
26
|
+
export declare const exportJobId: (input: {
|
|
27
|
+
readonly tenantId: string;
|
|
28
|
+
readonly exportId: string;
|
|
29
|
+
}) => string;
|
|
30
|
+
export declare const createBullMqExportDispatcher: (queue: ExportQueue, options?: JobDispatcherOptions) => ExportDispatcher;
|
|
31
|
+
//# sourceMappingURL=export.d.ts.map
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BullMQ `ExportDispatcher` (#134).
|
|
3
|
+
*
|
|
4
|
+
* A third queue, and for the same reason extraction got its own: a hundred-page PDF render must not sit in
|
|
5
|
+
* front of a user's next message, and rendering is CPU-bound where a run waits on a provider. The id, escaping
|
|
6
|
+
* and timeout behaviour are `dispatcher.ts`'s, reused rather than re-derived — the length-prefix fix exists
|
|
7
|
+
* because `${tenant}-${id}` is *ambiguous*, and a second implementation is a second chance to forget that.
|
|
8
|
+
*/
|
|
9
|
+
import { AgentPlatformError } from "../../core/errors.js";
|
|
10
|
+
import { QUEUE_ATTEMPTS, runJobId } from "./dispatcher.js";
|
|
11
|
+
/** Hyphen, not a colon: BullMQ rejects a queue name containing `:` outright. See `RUN_QUEUE_NAME`. */
|
|
12
|
+
export const EXPORT_QUEUE_NAME = "agentkit-exports";
|
|
13
|
+
export const EXPORT_JOB_NAME = "export";
|
|
14
|
+
/** Tenant-qualified and unambiguous, built by the same function a run's id is. */
|
|
15
|
+
export const exportJobId = (input) => runJobId({ tenantId: input.tenantId, runId: input.exportId });
|
|
16
|
+
const unavailable = (cause) => new AgentPlatformError({
|
|
17
|
+
code: "provider_unavailable",
|
|
18
|
+
message: "Could not enqueue the export: the job queue is unreachable",
|
|
19
|
+
retryable: true,
|
|
20
|
+
}, { cause });
|
|
21
|
+
export const createBullMqExportDispatcher = (queue, options = {}) => {
|
|
22
|
+
const timeoutMs = options.enqueueTimeoutMs ?? 5_000;
|
|
23
|
+
return {
|
|
24
|
+
async enqueueExport({ tenantId, exportId }) {
|
|
25
|
+
const add = queue.add(EXPORT_JOB_NAME, { tenantId, exportId },
|
|
26
|
+
// The dedup that matters: an export claimed once must be rendered once, and the store's unique
|
|
27
|
+
// constraint already guarantees one row per (artifact, version, format) — this stops a duplicated
|
|
28
|
+
// *message* becoming a second render of that row.
|
|
29
|
+
{ jobId: exportJobId({ tenantId, exportId }), attempts: QUEUE_ATTEMPTS });
|
|
30
|
+
let timer;
|
|
31
|
+
try {
|
|
32
|
+
await Promise.race([
|
|
33
|
+
add,
|
|
34
|
+
new Promise((_resolve, reject) => {
|
|
35
|
+
// A connection that is open-but-dead fails neither fast nor at all, so the timeout is part of the
|
|
36
|
+
// guarantee rather than a nicety.
|
|
37
|
+
timer = setTimeout(() => reject(unavailable(new Error(`enqueue timed out after ${timeoutMs}ms`))), timeoutMs);
|
|
38
|
+
}),
|
|
39
|
+
]);
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
throw error instanceof AgentPlatformError ? error : unavailable(error);
|
|
43
|
+
}
|
|
44
|
+
finally {
|
|
45
|
+
if (timer !== undefined)
|
|
46
|
+
clearTimeout(timer);
|
|
47
|
+
// The losing promise must not become an unhandled rejection when the timeout wins.
|
|
48
|
+
void Promise.resolve(add).catch(() => undefined);
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
};
|
|
53
|
+
//# sourceMappingURL=export.js.map
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BullMQ `ExtractionDispatcher` (#131) — the durable enqueue behind document extraction.
|
|
3
|
+
*
|
|
4
|
+
* Its own queue, not the run queue, and that is the decision worth stating. A shared queue would let a
|
|
5
|
+
* hundred-page PDF sit in front of a user's next message, which is precisely what AC-2 forbids; and the two
|
|
6
|
+
* kinds of work want different concurrency, because extraction is CPU-bound and a run is mostly waiting on a
|
|
7
|
+
* provider. Separate queues let a deployment give extraction one worker and runs ten.
|
|
8
|
+
*
|
|
9
|
+
* The id, escaping and timeout behaviour are `dispatcher.ts`'s, reused rather than re-derived: the ambiguity
|
|
10
|
+
* bug that `runJobId`'s length prefix fixes is a property of tenant-qualified ids in general, not of runs.
|
|
11
|
+
*/
|
|
12
|
+
import type { ExtractionDispatcher } from "../../documents/index.js";
|
|
13
|
+
import { type JobDispatcherOptions } from "./dispatcher.js";
|
|
14
|
+
/** Hyphen, not a colon: BullMQ rejects a queue name containing `:` outright. See `RUN_QUEUE_NAME`. */
|
|
15
|
+
export declare const EXTRACTION_QUEUE_NAME = "agentkit-extractions";
|
|
16
|
+
export declare const EXTRACTION_JOB_NAME = "extract";
|
|
17
|
+
export type ExtractionJobData = {
|
|
18
|
+
readonly tenantId: string;
|
|
19
|
+
readonly fileId: string;
|
|
20
|
+
};
|
|
21
|
+
/** The queue surface this adapter needs, structurally satisfied by BullMQ's `Queue`. */
|
|
22
|
+
export interface ExtractionQueue {
|
|
23
|
+
add(name: string, data: ExtractionJobData, opts?: {
|
|
24
|
+
readonly jobId?: string;
|
|
25
|
+
readonly attempts?: number;
|
|
26
|
+
}): Promise<unknown>;
|
|
27
|
+
close?(): Promise<void>;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* The job id, tenant-qualified and unambiguous.
|
|
31
|
+
*
|
|
32
|
+
* Deliberately the same construction as a run's — `runJobId` takes the second part under the name `runId`,
|
|
33
|
+
* and the file id goes there. Sharing it rather than writing a near-copy is the point: the length prefix
|
|
34
|
+
* exists because `${tenant}-${id}` is *ambiguous* (tenant `a-b`/file `c` and tenant `a`/file `b-c` collide),
|
|
35
|
+
* and a second implementation is a second chance to forget that.
|
|
36
|
+
*/
|
|
37
|
+
export declare const extractionJobId: (input: {
|
|
38
|
+
readonly tenantId: string;
|
|
39
|
+
readonly fileId: string;
|
|
40
|
+
}) => string;
|
|
41
|
+
export declare const createBullMqExtractionDispatcher: (queue: ExtractionQueue, options?: JobDispatcherOptions) => ExtractionDispatcher;
|
|
42
|
+
//# sourceMappingURL=extraction.d.ts.map
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BullMQ `ExtractionDispatcher` (#131) — the durable enqueue behind document extraction.
|
|
3
|
+
*
|
|
4
|
+
* Its own queue, not the run queue, and that is the decision worth stating. A shared queue would let a
|
|
5
|
+
* hundred-page PDF sit in front of a user's next message, which is precisely what AC-2 forbids; and the two
|
|
6
|
+
* kinds of work want different concurrency, because extraction is CPU-bound and a run is mostly waiting on a
|
|
7
|
+
* provider. Separate queues let a deployment give extraction one worker and runs ten.
|
|
8
|
+
*
|
|
9
|
+
* The id, escaping and timeout behaviour are `dispatcher.ts`'s, reused rather than re-derived: the ambiguity
|
|
10
|
+
* bug that `runJobId`'s length prefix fixes is a property of tenant-qualified ids in general, not of runs.
|
|
11
|
+
*/
|
|
12
|
+
import { AgentPlatformError } from "../../core/errors.js";
|
|
13
|
+
import { QUEUE_ATTEMPTS, runJobId } from "./dispatcher.js";
|
|
14
|
+
/** Hyphen, not a colon: BullMQ rejects a queue name containing `:` outright. See `RUN_QUEUE_NAME`. */
|
|
15
|
+
export const EXTRACTION_QUEUE_NAME = "agentkit-extractions";
|
|
16
|
+
export const EXTRACTION_JOB_NAME = "extract";
|
|
17
|
+
/**
|
|
18
|
+
* The job id, tenant-qualified and unambiguous.
|
|
19
|
+
*
|
|
20
|
+
* Deliberately the same construction as a run's — `runJobId` takes the second part under the name `runId`,
|
|
21
|
+
* and the file id goes there. Sharing it rather than writing a near-copy is the point: the length prefix
|
|
22
|
+
* exists because `${tenant}-${id}` is *ambiguous* (tenant `a-b`/file `c` and tenant `a`/file `b-c` collide),
|
|
23
|
+
* and a second implementation is a second chance to forget that.
|
|
24
|
+
*/
|
|
25
|
+
export const extractionJobId = (input) => runJobId({ tenantId: input.tenantId, runId: input.fileId });
|
|
26
|
+
const unavailable = (cause) => new AgentPlatformError({
|
|
27
|
+
code: "provider_unavailable",
|
|
28
|
+
message: "Could not enqueue the extraction: the job queue is unreachable",
|
|
29
|
+
retryable: true,
|
|
30
|
+
}, { cause });
|
|
31
|
+
export const createBullMqExtractionDispatcher = (queue, options = {}) => {
|
|
32
|
+
const timeoutMs = options.enqueueTimeoutMs ?? 5_000;
|
|
33
|
+
return {
|
|
34
|
+
async enqueueExtraction({ tenantId, fileId }) {
|
|
35
|
+
const add = queue.add(EXTRACTION_JOB_NAME, { tenantId, fileId },
|
|
36
|
+
// The dedup that matters here: an upload retried by a client must not extract the same file twice.
|
|
37
|
+
// `attempts: 1` for the same reason the run queue uses it — the caller owns retries, and multiplying
|
|
38
|
+
// two retry policies gives backoff neither layer intended.
|
|
39
|
+
{ jobId: extractionJobId({ tenantId, fileId }), attempts: QUEUE_ATTEMPTS });
|
|
40
|
+
let timer;
|
|
41
|
+
try {
|
|
42
|
+
await Promise.race([
|
|
43
|
+
add,
|
|
44
|
+
new Promise((_resolve, reject) => {
|
|
45
|
+
// A connection that is open-but-dead fails neither fast nor at all, so the timeout is part of
|
|
46
|
+
// the guarantee rather than a nicety.
|
|
47
|
+
timer = setTimeout(() => reject(unavailable(new Error(`enqueue timed out after ${timeoutMs}ms`))), timeoutMs);
|
|
48
|
+
}),
|
|
49
|
+
]);
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
throw error instanceof AgentPlatformError ? error : unavailable(error);
|
|
53
|
+
}
|
|
54
|
+
finally {
|
|
55
|
+
if (timer !== undefined)
|
|
56
|
+
clearTimeout(timer);
|
|
57
|
+
// The losing promise must not become an unhandled rejection when the timeout wins.
|
|
58
|
+
void Promise.resolve(add).catch(() => undefined);
|
|
59
|
+
}
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
};
|
|
63
|
+
//# sourceMappingURL=extraction.js.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BullMQ/Redis adapters — the durable job queue behind `JobDispatcher` (#105).
|
|
3
|
+
*
|
|
4
|
+
* `bullmq` and `ioredis` are imported only inside this directory, which is where the dependency
|
|
5
|
+
* boundary checker expects an adapter's client coupling to live.
|
|
6
|
+
*/
|
|
7
|
+
export * from "./dispatcher.js";
|
|
8
|
+
export * from "./extraction.js";
|
|
9
|
+
export * from "./export.js";
|
|
10
|
+
export * from "./queue.js";
|
|
11
|
+
export * from "./lock.js";
|
|
12
|
+
export * from "./consumer.js";
|
|
13
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BullMQ/Redis adapters — the durable job queue behind `JobDispatcher` (#105).
|
|
3
|
+
*
|
|
4
|
+
* `bullmq` and `ioredis` are imported only inside this directory, which is where the dependency
|
|
5
|
+
* boundary checker expects an adapter's client coupling to live.
|
|
6
|
+
*/
|
|
7
|
+
export * from "./dispatcher.js";
|
|
8
|
+
export * from "./extraction.js";
|
|
9
|
+
export * from "./export.js";
|
|
10
|
+
export * from "./queue.js";
|
|
11
|
+
export * from "./lock.js";
|
|
12
|
+
export * from "./consumer.js";
|
|
13
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import type { DistributedLockStore } from "../../runtime/index.js";
|
|
2
|
+
/**
|
|
3
|
+
* The Redis surface this lock needs, kept narrow deliberately.
|
|
4
|
+
*
|
|
5
|
+
* Not `ioredis`'s `Redis` directly: its `set` is heavily overloaded, so relying on structural
|
|
6
|
+
* compatibility would make an accidental argument-order change typecheck. `createIoredisLockClient`
|
|
7
|
+
* adapts it explicitly instead.
|
|
8
|
+
*/
|
|
9
|
+
export interface LockRedis {
|
|
10
|
+
/** `SET key value PX ttl NX` — returns "OK" when it took the lock, null when someone else holds it. */
|
|
11
|
+
setIfAbsent(key: string, value: string, ttlMs: number): Promise<string | null>;
|
|
12
|
+
/** `EVAL script numKeys key arg…` — returns the script's integer reply. */
|
|
13
|
+
runScript(script: string, key: string, args: readonly string[]): Promise<number>;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Release, guarded by the token.
|
|
17
|
+
*
|
|
18
|
+
* A plain `DEL` is the bug this exists to avoid: a holder whose lease expired *while it was still
|
|
19
|
+
* working* would delete its successor's lock, and two workers would then believe they hold the
|
|
20
|
+
* conversation. That turns "the lock is slow" into "the lock is broken", which is AC-2.
|
|
21
|
+
*/
|
|
22
|
+
export declare const RELEASE_SCRIPT = "\nif redis.call('get', KEYS[1]) == ARGV[1] then\n return redis.call('del', KEYS[1])\nend\nreturn 0";
|
|
23
|
+
/**
|
|
24
|
+
* Renew, guarded by the same token.
|
|
25
|
+
*
|
|
26
|
+
* `PEXPIRE` on its own is the same bug one step later: a stale holder would extend a lock it no longer
|
|
27
|
+
* owns, keeping the rightful holder out.
|
|
28
|
+
*/
|
|
29
|
+
export declare const RENEW_SCRIPT = "\nif redis.call('get', KEYS[1]) == ARGV[1] then\n return redis.call('pexpire', KEYS[1], ARGV[2])\nend\nreturn 0";
|
|
30
|
+
/** What the adapter actually returns. Wider than the port, which has no `renew`. */
|
|
31
|
+
export type LockHandle = {
|
|
32
|
+
/** Compare-and-delete. Safe to call twice; safe to call after expiry. */
|
|
33
|
+
readonly released: () => Promise<void>;
|
|
34
|
+
/** Compare-and-extend. `false` means the lock is no longer ours — stop working. */
|
|
35
|
+
readonly renew: (ttlMs?: number) => Promise<boolean>;
|
|
36
|
+
/** Unique per acquisition, not per worker. See `createRedisLockStore`. */
|
|
37
|
+
readonly token: string;
|
|
38
|
+
readonly key: string;
|
|
39
|
+
};
|
|
40
|
+
export type RedisLockOptions = {
|
|
41
|
+
/** Namespace, so a lock cannot collide with the queue's own keys. */
|
|
42
|
+
readonly keyPrefix?: string;
|
|
43
|
+
/** Injectable for tests. Production uses `crypto.randomUUID`. */
|
|
44
|
+
readonly newToken?: () => string;
|
|
45
|
+
};
|
|
46
|
+
export type RedisLockStore = DistributedLockStore & {
|
|
47
|
+
acquire(key: string, ttlMs: number): Promise<LockHandle | null>;
|
|
48
|
+
};
|
|
49
|
+
export declare const createRedisLockStore: (redis: LockRedis, options?: RedisLockOptions) => RedisLockStore;
|
|
50
|
+
/**
|
|
51
|
+
* Adapts `ioredis` explicitly. The argument order of `SET key value PX ttl NX` matters and is easy to
|
|
52
|
+
* transpose, so it is written once here rather than at every call site.
|
|
53
|
+
*/
|
|
54
|
+
export declare const createIoredisLockClient: (redis: {
|
|
55
|
+
set(...args: readonly (string | number)[]): Promise<string | null>;
|
|
56
|
+
eval(...args: readonly (string | number)[]): Promise<unknown>;
|
|
57
|
+
}) => LockRedis;
|
|
58
|
+
export type RenewalHandle = {
|
|
59
|
+
readonly stop: () => void;
|
|
60
|
+
readonly lost: () => boolean;
|
|
61
|
+
};
|
|
62
|
+
/**
|
|
63
|
+
* Heartbeat-driven renewal (AC-4).
|
|
64
|
+
*
|
|
65
|
+
* Renews on a timer and **stops the moment a renewal fails**. That is the important half: a renewal
|
|
66
|
+
* that returns false means the lock is no longer ours, and continuing to renew would be asserting
|
|
67
|
+
* ownership we lost. `lost()` lets the worker notice and stand down rather than carrying on with a
|
|
68
|
+
* lock it does not hold — the same shape as the worker's existing `ClaimLostError` path.
|
|
69
|
+
*
|
|
70
|
+
* Renews at a fraction of the TTL rather than at the TTL, so one slow round trip does not drop a lock
|
|
71
|
+
* that is still legitimately held.
|
|
72
|
+
*/
|
|
73
|
+
export declare const startLockRenewal: (handle: LockHandle, options: {
|
|
74
|
+
readonly ttlMs: number;
|
|
75
|
+
readonly everyMs?: number;
|
|
76
|
+
}) => RenewalHandle;
|
|
77
|
+
//# sourceMappingURL=lock.d.ts.map
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Redis `DistributedLockStore` (#106) — per-conversation mutual exclusion across worker processes.
|
|
3
|
+
*
|
|
4
|
+
* **Read this before relying on it for safety, because you should not.** This is a single-instance
|
|
5
|
+
* lock. Under a Redis failover, a network partition, or a clock jump, two holders are possible. It is
|
|
6
|
+
* *not* a consensus lock and no amount of care in this file makes it one.
|
|
7
|
+
*
|
|
8
|
+
* The safety property lives in the database, where #98 put it: the run coordinator's slot table with
|
|
9
|
+
* `SELECT … FOR UPDATE`, and `RunStore`'s lease compare-and-set. The worker already says as much where
|
|
10
|
+
* it declares this dependency — *"Optional belt-and-suspenders mutual exclusion around the atomic
|
|
11
|
+
* claim"* — and `locks?` is optional precisely so a deployment can run without it.
|
|
12
|
+
*
|
|
13
|
+
* What this buys is **contention**: workers that would otherwise all pile onto the same slot row back
|
|
14
|
+
* off at Redis instead. That is worth having on a hot path, and it is all it is worth.
|
|
15
|
+
*/
|
|
16
|
+
import { randomUUID } from "node:crypto";
|
|
17
|
+
/**
|
|
18
|
+
* Release, guarded by the token.
|
|
19
|
+
*
|
|
20
|
+
* A plain `DEL` is the bug this exists to avoid: a holder whose lease expired *while it was still
|
|
21
|
+
* working* would delete its successor's lock, and two workers would then believe they hold the
|
|
22
|
+
* conversation. That turns "the lock is slow" into "the lock is broken", which is AC-2.
|
|
23
|
+
*/
|
|
24
|
+
export const RELEASE_SCRIPT = `
|
|
25
|
+
if redis.call('get', KEYS[1]) == ARGV[1] then
|
|
26
|
+
return redis.call('del', KEYS[1])
|
|
27
|
+
end
|
|
28
|
+
return 0`;
|
|
29
|
+
/**
|
|
30
|
+
* Renew, guarded by the same token.
|
|
31
|
+
*
|
|
32
|
+
* `PEXPIRE` on its own is the same bug one step later: a stale holder would extend a lock it no longer
|
|
33
|
+
* owns, keeping the rightful holder out.
|
|
34
|
+
*/
|
|
35
|
+
export const RENEW_SCRIPT = `
|
|
36
|
+
if redis.call('get', KEYS[1]) == ARGV[1] then
|
|
37
|
+
return redis.call('pexpire', KEYS[1], ARGV[2])
|
|
38
|
+
end
|
|
39
|
+
return 0`;
|
|
40
|
+
export const createRedisLockStore = (redis, options = {}) => {
|
|
41
|
+
const prefix = options.keyPrefix ?? "agentkit-lock:";
|
|
42
|
+
// Per *acquisition*, not per worker. A worker that acquires, loses the lease, and re-acquires must
|
|
43
|
+
// not be able to release the intervening holder's lock with a handle it kept from last time.
|
|
44
|
+
const newToken = options.newToken ?? (() => randomUUID());
|
|
45
|
+
return {
|
|
46
|
+
async acquire(key, ttlMs) {
|
|
47
|
+
const namespaced = `${prefix}${key}`;
|
|
48
|
+
const token = newToken();
|
|
49
|
+
// NX is the mutual exclusion; PX is AC-3. Together they mean a dead holder's lock frees itself
|
|
50
|
+
// with no reaper, no operator, and no second liveness clock to drift out of step.
|
|
51
|
+
const acquired = await redis.setIfAbsent(namespaced, token, ttlMs);
|
|
52
|
+
if (acquired === null)
|
|
53
|
+
return null;
|
|
54
|
+
let releasedAlready = false;
|
|
55
|
+
return {
|
|
56
|
+
token,
|
|
57
|
+
key: namespaced,
|
|
58
|
+
async released() {
|
|
59
|
+
// Idempotent: a caller that releases in a `finally` after an error path may well call twice,
|
|
60
|
+
// and the second call must not become a delete of whatever holds the key by then.
|
|
61
|
+
if (releasedAlready)
|
|
62
|
+
return;
|
|
63
|
+
releasedAlready = true;
|
|
64
|
+
await redis.runScript(RELEASE_SCRIPT, namespaced, [token]);
|
|
65
|
+
},
|
|
66
|
+
async renew(nextTtlMs = ttlMs) {
|
|
67
|
+
if (releasedAlready)
|
|
68
|
+
return false;
|
|
69
|
+
const extended = await redis.runScript(RENEW_SCRIPT, namespaced, [token, String(nextTtlMs)]);
|
|
70
|
+
return extended === 1;
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
};
|
|
76
|
+
/**
|
|
77
|
+
* Adapts `ioredis` explicitly. The argument order of `SET key value PX ttl NX` matters and is easy to
|
|
78
|
+
* transpose, so it is written once here rather than at every call site.
|
|
79
|
+
*/
|
|
80
|
+
export const createIoredisLockClient = (redis) => ({
|
|
81
|
+
async setIfAbsent(key, value, ttlMs) {
|
|
82
|
+
return redis.set(key, value, "PX", Math.max(1, Math.floor(ttlMs)), "NX");
|
|
83
|
+
},
|
|
84
|
+
async runScript(script, key, args) {
|
|
85
|
+
const reply = await redis.eval(script, 1, key, ...args);
|
|
86
|
+
return Number(reply ?? 0);
|
|
87
|
+
},
|
|
88
|
+
});
|
|
89
|
+
/**
|
|
90
|
+
* Heartbeat-driven renewal (AC-4).
|
|
91
|
+
*
|
|
92
|
+
* Renews on a timer and **stops the moment a renewal fails**. That is the important half: a renewal
|
|
93
|
+
* that returns false means the lock is no longer ours, and continuing to renew would be asserting
|
|
94
|
+
* ownership we lost. `lost()` lets the worker notice and stand down rather than carrying on with a
|
|
95
|
+
* lock it does not hold — the same shape as the worker's existing `ClaimLostError` path.
|
|
96
|
+
*
|
|
97
|
+
* Renews at a fraction of the TTL rather than at the TTL, so one slow round trip does not drop a lock
|
|
98
|
+
* that is still legitimately held.
|
|
99
|
+
*/
|
|
100
|
+
export const startLockRenewal = (handle, options) => {
|
|
101
|
+
const everyMs = options.everyMs ?? Math.max(1, Math.floor(options.ttlMs / 3));
|
|
102
|
+
let lost = false;
|
|
103
|
+
const timer = setInterval(() => {
|
|
104
|
+
void handle
|
|
105
|
+
.renew(options.ttlMs)
|
|
106
|
+
.then((ok) => {
|
|
107
|
+
if (!ok) {
|
|
108
|
+
lost = true;
|
|
109
|
+
clearInterval(timer);
|
|
110
|
+
}
|
|
111
|
+
})
|
|
112
|
+
.catch(() => {
|
|
113
|
+
// A failed round trip is not proof the lock is gone, but it is not proof it is held either.
|
|
114
|
+
// Treated as lost, because the alternative is a worker that keeps working on a maybe.
|
|
115
|
+
lost = true;
|
|
116
|
+
clearInterval(timer);
|
|
117
|
+
});
|
|
118
|
+
}, everyMs);
|
|
119
|
+
// Never hold the process open: a renewal timer must not be the reason a worker cannot exit.
|
|
120
|
+
timer.unref?.();
|
|
121
|
+
return {
|
|
122
|
+
stop: () => clearInterval(timer),
|
|
123
|
+
lost: () => lost,
|
|
124
|
+
};
|
|
125
|
+
};
|
|
126
|
+
//# sourceMappingURL=lock.js.map
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The real BullMQ queue and its connection policy (#105).
|
|
3
|
+
*
|
|
4
|
+
* Separate from `dispatcher.ts` so the dispatcher stays testable without Redis, and so the connection
|
|
5
|
+
* decisions — which are the interesting part — sit in one place with their reasons.
|
|
6
|
+
*/
|
|
7
|
+
import { Queue } from "bullmq";
|
|
8
|
+
import { Redis } from "ioredis";
|
|
9
|
+
import { type JobQueue } from "./dispatcher.js";
|
|
10
|
+
export type RunQueueOptions = {
|
|
11
|
+
readonly url: string;
|
|
12
|
+
/** Ceiling on establishing a connection. */
|
|
13
|
+
readonly connectTimeoutMs?: number;
|
|
14
|
+
/** Completed/failed jobs to retain. Retained, not zero — see below. */
|
|
15
|
+
readonly keepCompleted?: number;
|
|
16
|
+
readonly keepFailed?: number;
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* `ioredis` options chosen for a *producer*, which wants to fail fast.
|
|
20
|
+
*
|
|
21
|
+
* `enableOfflineQueue: false` is the one AC-4 turns on. By default ioredis buffers commands while
|
|
22
|
+
* disconnected, so `queue.add()` would sit there until the connection came back — the hang the AC
|
|
23
|
+
* names. With it off, the command rejects immediately and the dispatcher can wrap that into a typed
|
|
24
|
+
* error the API layer can turn into a real response.
|
|
25
|
+
*
|
|
26
|
+
* `maxRetriesPerRequest: 1` for the same reason: a producer would rather tell the caller now than
|
|
27
|
+
* retry silently behind a request that is already waiting. Note this is deliberately **not** the right
|
|
28
|
+
* setting for a BullMQ *worker* connection, which needs `maxRetriesPerRequest: null` for its blocking
|
|
29
|
+
* reads — a worker built here would need its own connection, not this one.
|
|
30
|
+
*
|
|
31
|
+
* `retryStrategy` backs off with jitter and gives up climbing at 2s, so a flapping Redis produces a
|
|
32
|
+
* steady trickle of reconnects rather than a thundering herd from every process at once.
|
|
33
|
+
*/
|
|
34
|
+
export declare const createRunQueueConnection: (options: RunQueueOptions) => Redis;
|
|
35
|
+
/**
|
|
36
|
+
* The run queue.
|
|
37
|
+
*
|
|
38
|
+
* `removeOnComplete` keeps a bounded history rather than zero. Two reasons: AC-5's counts are more
|
|
39
|
+
* useful with a completed window, and a zero-retention queue makes it tempting to read `jobId` dedup
|
|
40
|
+
* as durable idempotency — it is not, since a removed job's id is immediately reusable. What stops a
|
|
41
|
+
* finished run being re-executed is `RunStore`, not this.
|
|
42
|
+
*
|
|
43
|
+
* `attempts` comes from the shared constant so the queue and the dispatcher cannot disagree about
|
|
44
|
+
* whose retry policy is in force (AC-3).
|
|
45
|
+
*/
|
|
46
|
+
export declare const createBullMqRunQueue: (options: RunQueueOptions) => JobQueue & {
|
|
47
|
+
readonly queue: Queue;
|
|
48
|
+
close(): Promise<void>;
|
|
49
|
+
};
|
|
50
|
+
//# sourceMappingURL=queue.d.ts.map
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The real BullMQ queue and its connection policy (#105).
|
|
3
|
+
*
|
|
4
|
+
* Separate from `dispatcher.ts` so the dispatcher stays testable without Redis, and so the connection
|
|
5
|
+
* decisions — which are the interesting part — sit in one place with their reasons.
|
|
6
|
+
*/
|
|
7
|
+
import { Queue } from "bullmq";
|
|
8
|
+
import { Redis } from "ioredis";
|
|
9
|
+
import { QUEUE_ATTEMPTS, RUN_QUEUE_NAME } from "./dispatcher.js";
|
|
10
|
+
/**
|
|
11
|
+
* `ioredis` options chosen for a *producer*, which wants to fail fast.
|
|
12
|
+
*
|
|
13
|
+
* `enableOfflineQueue: false` is the one AC-4 turns on. By default ioredis buffers commands while
|
|
14
|
+
* disconnected, so `queue.add()` would sit there until the connection came back — the hang the AC
|
|
15
|
+
* names. With it off, the command rejects immediately and the dispatcher can wrap that into a typed
|
|
16
|
+
* error the API layer can turn into a real response.
|
|
17
|
+
*
|
|
18
|
+
* `maxRetriesPerRequest: 1` for the same reason: a producer would rather tell the caller now than
|
|
19
|
+
* retry silently behind a request that is already waiting. Note this is deliberately **not** the right
|
|
20
|
+
* setting for a BullMQ *worker* connection, which needs `maxRetriesPerRequest: null` for its blocking
|
|
21
|
+
* reads — a worker built here would need its own connection, not this one.
|
|
22
|
+
*
|
|
23
|
+
* `retryStrategy` backs off with jitter and gives up climbing at 2s, so a flapping Redis produces a
|
|
24
|
+
* steady trickle of reconnects rather than a thundering herd from every process at once.
|
|
25
|
+
*/
|
|
26
|
+
export const createRunQueueConnection = (options) => new Redis(options.url, {
|
|
27
|
+
enableOfflineQueue: false,
|
|
28
|
+
maxRetriesPerRequest: 1,
|
|
29
|
+
connectTimeout: options.connectTimeoutMs ?? 3_000,
|
|
30
|
+
lazyConnect: true,
|
|
31
|
+
retryStrategy: (attempt) => {
|
|
32
|
+
const base = Math.min(2_000, 100 * 2 ** Math.min(attempt, 5));
|
|
33
|
+
// Deterministic-ish jitter without Math.random, so a retry schedule is reproducible from the
|
|
34
|
+
// attempt number when reading logs.
|
|
35
|
+
return base - (base * (attempt % 4)) / 16;
|
|
36
|
+
},
|
|
37
|
+
});
|
|
38
|
+
/**
|
|
39
|
+
* The run queue.
|
|
40
|
+
*
|
|
41
|
+
* `removeOnComplete` keeps a bounded history rather than zero. Two reasons: AC-5's counts are more
|
|
42
|
+
* useful with a completed window, and a zero-retention queue makes it tempting to read `jobId` dedup
|
|
43
|
+
* as durable idempotency — it is not, since a removed job's id is immediately reusable. What stops a
|
|
44
|
+
* finished run being re-executed is `RunStore`, not this.
|
|
45
|
+
*
|
|
46
|
+
* `attempts` comes from the shared constant so the queue and the dispatcher cannot disagree about
|
|
47
|
+
* whose retry policy is in force (AC-3).
|
|
48
|
+
*/
|
|
49
|
+
export const createBullMqRunQueue = (options) => {
|
|
50
|
+
const connection = createRunQueueConnection(options);
|
|
51
|
+
const queue = new Queue(RUN_QUEUE_NAME, {
|
|
52
|
+
connection,
|
|
53
|
+
defaultJobOptions: {
|
|
54
|
+
attempts: QUEUE_ATTEMPTS,
|
|
55
|
+
removeOnComplete: { count: options.keepCompleted ?? 1_000 },
|
|
56
|
+
removeOnFail: { count: options.keepFailed ?? 5_000 },
|
|
57
|
+
},
|
|
58
|
+
});
|
|
59
|
+
return {
|
|
60
|
+
queue,
|
|
61
|
+
add: (name, data, opts) => queue.add(name, data, opts),
|
|
62
|
+
getJobCounts: (...types) => queue.getJobCounts(...types),
|
|
63
|
+
/**
|
|
64
|
+
* Exposed so the dispatcher can clear a **finished** job holding a reusable id (#156).
|
|
65
|
+
*
|
|
66
|
+
* This object deliberately narrows the BullMQ `Queue` to the few methods the port needs, which is right — but
|
|
67
|
+
* it meant adding `getJob` to `JobQueue` and to the dispatcher changed nothing at all, because the method was
|
|
68
|
+
* never on the object the dispatcher received. The fix type-checked, the test against a raw `Queue` passed,
|
|
69
|
+
* and the real deployment silently kept the old behaviour: every approval resume still sat in `queued`.
|
|
70
|
+
*
|
|
71
|
+
* A narrowing wrapper is a second place every capability has to be added, and forgetting is invisible.
|
|
72
|
+
*/
|
|
73
|
+
getJob: (jobId) => queue.getJob(jobId),
|
|
74
|
+
async close() {
|
|
75
|
+
await queue.close();
|
|
76
|
+
// The queue closes its own connection only when it created it; this one was injected.
|
|
77
|
+
await connection.quit().catch(() => undefined);
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
};
|
|
81
|
+
//# sourceMappingURL=queue.js.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* In-memory `ArtifactExportStore` — the reference implementation (#134).
|
|
3
|
+
*
|
|
4
|
+
* The only interesting behaviour is `claim`, and it is interesting for one reason: **it must be a claim, not
|
|
5
|
+
* an insert.** Two requests for the same PDF must produce one render, and the second must be told so rather
|
|
6
|
+
* than starting a duplicate. A blind insert would render the same document twice and leave two rows pointing
|
|
7
|
+
* at two identical files.
|
|
8
|
+
*/
|
|
9
|
+
import type { ArtifactExportStore } from "../../persistence/index.js";
|
|
10
|
+
export declare const createMemoryArtifactExportStore: () => ArtifactExportStore;
|
|
11
|
+
//# sourceMappingURL=artifact-exports.d.ts.map
|