@retinue/agentkit 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -277
- package/dist/adapters/audio/openai.d.ts +36 -0
- package/dist/adapters/audio/openai.js +136 -0
- package/dist/adapters/bullmq/lock.d.ts +26 -2
- package/dist/adapters/embeddings/openai.d.ts +45 -0
- package/dist/adapters/embeddings/openai.js +109 -0
- package/dist/adapters/memory/connections.d.ts +10 -0
- package/dist/adapters/memory/connections.js +87 -0
- package/dist/adapters/memory/graph.d.ts +17 -0
- package/dist/adapters/memory/graph.js +267 -0
- package/dist/adapters/memory/index.d.ts +3 -0
- package/dist/adapters/memory/index.js +3 -0
- package/dist/adapters/memory/rate-limit.d.ts +14 -0
- package/dist/adapters/memory/rate-limit.js +28 -0
- package/dist/adapters/memory/runtime.js +0 -0
- package/dist/adapters/postgres/connections.d.ts +15 -0
- package/dist/adapters/postgres/connections.js +134 -0
- package/dist/adapters/postgres/graph.d.ts +24 -0
- package/dist/adapters/postgres/graph.js +325 -0
- package/dist/adapters/postgres/index.d.ts +2 -0
- package/dist/adapters/postgres/index.js +2 -0
- package/dist/adapters/postgres/migrations.js +198 -0
- package/dist/adapters/postgres/run-store.js +26 -2
- package/dist/adapters/postgres/schema.d.ts +61 -2
- package/dist/adapters/postgres/schema.js +95 -10
- package/dist/adapters/redis/index.d.ts +1 -0
- package/dist/adapters/redis/index.js +1 -0
- package/dist/adapters/redis/rate-limit.d.ts +46 -0
- package/dist/adapters/redis/rate-limit.js +55 -0
- package/dist/adapters/supabase/index.d.ts +15 -0
- package/dist/adapters/supabase/index.js +15 -0
- package/dist/adapters/supabase/rls.js +33 -0
- package/dist/agents/agent.d.ts +99 -1
- package/dist/agents/agent.js +215 -16
- package/dist/agents/define.js +24 -4
- package/dist/agents/engine.d.ts +48 -0
- package/dist/agents/engine.js +370 -13
- package/dist/audio/index.d.ts +151 -0
- package/dist/audio/index.js +183 -0
- package/dist/capabilities/index.d.ts +5 -1
- package/dist/capabilities/index.js +23 -0
- package/dist/capabilities/runtime.d.ts +8 -0
- package/dist/connections/cipher.d.ts +103 -0
- package/dist/connections/cipher.js +141 -0
- package/dist/connections/index.d.ts +156 -0
- package/dist/connections/index.js +72 -0
- package/dist/connections/oauth/client.d.ts +84 -0
- package/dist/connections/oauth/client.js +141 -0
- package/dist/connections/oauth/index.d.ts +166 -0
- package/dist/connections/oauth/index.js +226 -0
- package/dist/connections/oauth/service.d.ts +90 -0
- package/dist/connections/oauth/service.js +132 -0
- package/dist/connections/pause.d.ts +68 -0
- package/dist/connections/pause.js +97 -0
- package/dist/connections/resolver.d.ts +59 -0
- package/dist/connections/resolver.js +138 -0
- package/dist/connections/resume.d.ts +64 -0
- package/dist/connections/resume.js +99 -0
- package/dist/connections/vault-cipher.d.ts +95 -0
- package/dist/connections/vault-cipher.js +206 -0
- package/dist/core/budget.d.ts +55 -0
- package/dist/core/budget.js +56 -0
- package/dist/core/content-parts.d.ts +26 -2
- package/dist/core/content-parts.js +1 -0
- package/dist/core/context.d.ts +25 -0
- package/dist/core/errors.d.ts +1 -1
- package/dist/core/errors.js +15 -0
- package/dist/core/events.d.ts +110 -2
- package/dist/core/events.js +33 -2
- package/dist/core/index.d.ts +1 -0
- package/dist/core/index.js +1 -0
- package/dist/core/validation.js +19 -0
- package/dist/documents/index.d.ts +14 -0
- package/dist/documents/parsers/text.d.ts +16 -0
- package/dist/documents/parsers/text.js +54 -2
- package/dist/entries/connections.d.ts +10 -0
- package/dist/entries/connections.js +10 -0
- package/dist/entries/guardrails.d.ts +14 -0
- package/dist/entries/guardrails.js +14 -0
- package/dist/entries/knowledge.d.ts +17 -0
- package/dist/entries/knowledge.js +16 -0
- package/dist/entries/mcp-server.d.ts +9 -0
- package/dist/entries/mcp-server.js +9 -0
- package/dist/entries/runtime.d.ts +10 -0
- package/dist/entries/runtime.js +8 -0
- package/dist/entries/testing.d.ts +28 -0
- package/dist/entries/testing.js +28 -0
- package/dist/graphql/resolvers.d.ts +7 -1
- package/dist/graphql/resolvers.js +20 -0
- package/dist/graphql/schema.d.ts +1 -1
- package/dist/graphql/schema.js +44 -0
- package/dist/guardrails/index.d.ts +115 -0
- package/dist/guardrails/index.js +108 -0
- package/dist/guardrails/moderation.d.ts +53 -0
- package/dist/guardrails/moderation.js +75 -0
- package/dist/guardrails/pii.d.ts +75 -0
- package/dist/guardrails/pii.js +193 -0
- package/dist/knowledge/communities.d.ts +166 -0
- package/dist/knowledge/communities.js +377 -0
- package/dist/knowledge/graph-global.d.ts +153 -0
- package/dist/knowledge/graph-global.js +246 -0
- package/dist/knowledge/graph-retrieval.d.ts +150 -0
- package/dist/knowledge/graph-retrieval.js +303 -0
- package/dist/knowledge/graph.d.ts +223 -0
- package/dist/knowledge/graph.js +369 -0
- package/dist/knowledge/index.d.ts +25 -0
- package/dist/knowledge/index.js +37 -1
- package/dist/knowledge/navigate.d.ts +89 -0
- package/dist/knowledge/navigate.js +107 -0
- package/dist/knowledge/retrieval.d.ts +98 -5
- package/dist/knowledge/retrieval.js +158 -28
- package/dist/mcp-server/index.d.ts +168 -0
- package/dist/mcp-server/index.js +175 -0
- package/dist/models/extraction.d.ts +73 -0
- package/dist/models/extraction.js +118 -0
- package/dist/models/index.d.ts +40 -1
- package/dist/models/index.js +22 -1
- package/dist/models/pricing.d.ts +2 -0
- package/dist/models/pricing.js +17 -1
- package/dist/models/provider-factory.js +9 -7
- package/dist/models/streaming.d.ts +111 -1
- package/dist/models/streaming.js +184 -3
- package/dist/persistence/index.d.ts +230 -0
- package/dist/persistence/index.js +2 -0
- package/dist/runtime/index.d.ts +1 -1
- package/dist/runtime/index.js +12 -0
- package/dist/runtime/worker.d.ts +36 -1
- package/dist/runtime/worker.js +37 -4
- package/dist/security/checklist.js +55 -0
- package/dist/security/findings.js +18 -9
- package/dist/server/bin.d.ts +19 -0
- package/dist/server/bin.js +226 -0
- package/dist/server/boot.d.ts +20 -2
- package/dist/server/boot.js +29 -3
- package/dist/server/cli.d.ts +13 -0
- package/dist/server/cli.js +7 -4
- package/dist/server/doctor.d.ts +74 -0
- package/dist/server/doctor.js +280 -0
- package/dist/skills/catalogue.d.ts +49 -0
- package/dist/skills/catalogue.js +61 -0
- package/dist/skills/context.d.ts +61 -0
- package/dist/skills/context.js +107 -0
- package/dist/skills/index.d.ts +2 -0
- package/dist/skills/index.js +2 -0
- package/dist/telemetry/spans.js +17 -0
- package/dist/testing/conformance/artifact-exports.d.ts +20 -0
- package/dist/testing/conformance/artifact-exports.js +172 -0
- package/dist/testing/conformance/artifacts.d.ts +11 -0
- package/dist/testing/conformance/artifacts.js +316 -0
- package/dist/testing/conformance/audio.d.ts +38 -0
- package/dist/testing/conformance/audio.js +145 -0
- package/dist/testing/conformance/capability.d.ts +23 -0
- package/dist/testing/conformance/capability.js +27 -0
- package/dist/testing/conformance/checkpoint-store.d.ts +15 -0
- package/dist/testing/conformance/checkpoint-store.js +72 -0
- package/dist/testing/conformance/connections.d.ts +14 -0
- package/dist/testing/conformance/connections.js +171 -0
- package/dist/testing/conformance/conversation-store.d.ts +10 -0
- package/dist/testing/conformance/conversation-store.js +60 -0
- package/dist/testing/conformance/evaluation.d.ts +10 -0
- package/dist/testing/conformance/evaluation.js +295 -0
- package/dist/testing/conformance/files.d.ts +19 -0
- package/dist/testing/conformance/files.js +454 -0
- package/dist/testing/conformance/flows.d.ts +16 -0
- package/dist/testing/conformance/flows.js +193 -0
- package/dist/testing/conformance/graph.d.ts +22 -0
- package/dist/testing/conformance/graph.js +500 -0
- package/dist/testing/conformance/hitl.d.ts +25 -0
- package/dist/testing/conformance/hitl.js +523 -0
- package/dist/testing/conformance/index.d.ts +163 -0
- package/dist/testing/conformance/index.js +321 -0
- package/dist/testing/conformance/invariants.d.ts +23 -0
- package/dist/testing/conformance/invariants.js +80 -0
- package/dist/testing/conformance/knowledge.d.ts +41 -0
- package/dist/testing/conformance/knowledge.js +592 -0
- package/dist/testing/conformance/parents.d.ts +50 -0
- package/dist/testing/conformance/parents.js +39 -0
- package/dist/testing/conformance/rate-limit.d.ts +25 -0
- package/dist/testing/conformance/rate-limit.js +71 -0
- package/dist/testing/conformance/records.d.ts +36 -0
- package/dist/testing/conformance/records.js +400 -0
- package/dist/testing/conformance/rollups.d.ts +30 -0
- package/dist/testing/conformance/rollups.js +425 -0
- package/dist/testing/conformance/run-coordinator.d.ts +27 -0
- package/dist/testing/conformance/run-coordinator.js +150 -0
- package/dist/testing/conformance/run-event-log.d.ts +9 -0
- package/dist/testing/conformance/run-event-log.js +138 -0
- package/dist/testing/conformance/run-store.d.ts +12 -0
- package/dist/testing/conformance/run-store.js +294 -0
- package/dist/testing/conformance/session-state.d.ts +38 -0
- package/dist/testing/conformance/session-state.js +198 -0
- package/dist/testing/conformance/usage-limits.d.ts +10 -0
- package/dist/testing/conformance/usage-limits.js +309 -0
- package/dist/testing/conformance.d.ts +8 -0
- package/dist/testing/conformance.js +8 -0
- package/dist/testing/memory-backend.d.ts +39 -0
- package/dist/testing/memory-backend.js +41 -0
- package/dist/testing/pglite.d.ts +55 -0
- package/dist/testing/pglite.js +78 -0
- package/dist/testing/stub-model.d.ts +58 -0
- package/dist/testing/stub-model.js +71 -0
- package/dist/testing/supabase-storage-double.d.ts +35 -0
- package/dist/testing/supabase-storage-double.js +101 -0
- package/dist/toolkit/files.d.ts +125 -0
- package/dist/toolkit/files.js +320 -0
- package/dist/toolkit/http.d.ts +27 -0
- package/dist/toolkit/http.js +48 -0
- package/dist/toolkit/index.d.ts +31 -1
- package/dist/toolkit/index.js +26 -1
- package/dist/toolkit/mime.d.ts +97 -0
- package/dist/toolkit/mime.js +233 -0
- package/dist/toolkit/sandbox.d.ts +119 -0
- package/dist/toolkit/sandbox.js +239 -0
- package/dist/toolkit/ssrf.d.ts +139 -0
- package/dist/toolkit/ssrf.js +349 -0
- package/dist/toolkit/vendor.d.ts +91 -0
- package/dist/toolkit/vendor.js +153 -0
- package/dist/toolkit/web.d.ts +13 -0
- package/dist/toolkit/web.js +7 -1
- package/dist/tools/budget.d.ts +28 -0
- package/dist/tools/budget.js +35 -0
- package/dist/tools/credentials.d.ts +316 -0
- package/dist/tools/credentials.js +369 -0
- package/dist/tools/define.d.ts +45 -0
- package/dist/tools/define.js +26 -0
- package/dist/tools/find.d.ts +109 -0
- package/dist/tools/find.js +210 -0
- package/dist/tools/index.d.ts +35 -2
- package/dist/tools/index.js +11 -0
- package/dist/tools/library/audio.d.ts +63 -0
- package/dist/tools/library/audio.js +148 -0
- package/dist/tools/library/fs.d.ts +24 -0
- package/dist/tools/library/fs.js +102 -0
- package/dist/tools/library/index.d.ts +37 -2
- package/dist/tools/library/index.js +47 -0
- package/dist/tools/library/shell.d.ts +45 -0
- package/dist/tools/library/shell.js +70 -0
- package/dist/tools/meta-tools.js +8 -0
- package/dist/tools/registry.d.ts +146 -0
- package/dist/tools/registry.js +250 -4
- package/dist/usage/index.d.ts +21 -0
- package/dist/usage/index.js +1 -0
- package/dist/usage/rate-limit.d.ts +122 -0
- package/dist/usage/rate-limit.js +131 -0
- package/dist/usage/recorder.js +3 -0
- package/package.json +33 -2
|
@@ -1419,6 +1419,204 @@ export const MIGRATIONS = [
|
|
|
1419
1419
|
`ALTER TABLE runs DROP COLUMN IF EXISTS input`,
|
|
1420
1420
|
],
|
|
1421
1421
|
},
|
|
1422
|
+
{
|
|
1423
|
+
/**
|
|
1424
|
+
* A tenant's connections to third-party providers — REQ-063 (#259), task #261.
|
|
1425
|
+
*
|
|
1426
|
+
* **Every secret column holds ciphertext.** The table is deliberately shaped so that only
|
|
1427
|
+
* `secret_ciphertext` could ever contain one: the username of a basic credential, the header name of a
|
|
1428
|
+
* custom-header one and every vendor identifier live in `metadata`, readable, because a connection list is
|
|
1429
|
+
* a screen a person looks at and rendering it must not need a key.
|
|
1430
|
+
*
|
|
1431
|
+
* `key_id` is what makes rotation possible. A design without it has a first key that is permanent:
|
|
1432
|
+
* re-sealing every row needs the old key, which a process that has retired it no longer has.
|
|
1433
|
+
*
|
|
1434
|
+
* `revoked_at` rather than a delete, because "who connected this and when was it removed" is a question a
|
|
1435
|
+
* security review asks. `purge` is the one hard delete, and it exists for `docs/18`: a soft-deleted
|
|
1436
|
+
* credential is still a credential, and "we kept it for the audit trail" is not an answer to "delete my
|
|
1437
|
+
* data".
|
|
1438
|
+
*/
|
|
1439
|
+
id: "0032_connections",
|
|
1440
|
+
up: [
|
|
1441
|
+
`CREATE TABLE IF NOT EXISTS connections (
|
|
1442
|
+
tenant_id text NOT NULL,
|
|
1443
|
+
id text NOT NULL,
|
|
1444
|
+
provider text NOT NULL,
|
|
1445
|
+
label text,
|
|
1446
|
+
mode text NOT NULL,
|
|
1447
|
+
scheme text NOT NULL,
|
|
1448
|
+
metadata jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
1449
|
+
granted_scopes text[],
|
|
1450
|
+
secret_key_id text NOT NULL,
|
|
1451
|
+
secret_algorithm text NOT NULL,
|
|
1452
|
+
secret_nonce text NOT NULL,
|
|
1453
|
+
secret_ciphertext text NOT NULL,
|
|
1454
|
+
expires_at timestamptz,
|
|
1455
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
1456
|
+
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
1457
|
+
revoked_at timestamptz,
|
|
1458
|
+
PRIMARY KEY (tenant_id, id)
|
|
1459
|
+
)`,
|
|
1460
|
+
// Listing a tenant's live connections for one provider is the hot path: it happens on every resolution
|
|
1461
|
+
// of a bare `<provider>` ref.
|
|
1462
|
+
`CREATE INDEX IF NOT EXISTS connections_tenant_provider_idx
|
|
1463
|
+
ON connections (tenant_id, provider, created_at)
|
|
1464
|
+
WHERE revoked_at IS NULL`,
|
|
1465
|
+
],
|
|
1466
|
+
down: [`DROP TABLE IF EXISTS connections`],
|
|
1467
|
+
},
|
|
1468
|
+
{
|
|
1469
|
+
/**
|
|
1470
|
+
* A tenant's *own* OAuth app registration, in the same table — task #263.
|
|
1471
|
+
*
|
|
1472
|
+
* Same shape as a connection in every respect that matters: tenant-scoped, provider-keyed, one sealed
|
|
1473
|
+
* secret, metadata beside it, revocable, and covered by the same retention promise. A discriminator rather
|
|
1474
|
+
* than a second table means one cipher, one RLS policy, one retention path and one conformance suite.
|
|
1475
|
+
*
|
|
1476
|
+
* Defaulted to `'connection'`, so every existing row keeps meaning what it meant.
|
|
1477
|
+
*/
|
|
1478
|
+
id: "0033_connection_kind",
|
|
1479
|
+
up: [
|
|
1480
|
+
`ALTER TABLE connections ADD COLUMN IF NOT EXISTS kind text NOT NULL DEFAULT 'connection'`,
|
|
1481
|
+
// A tenant has at most one registered app per provider; two would make "which client issued this token"
|
|
1482
|
+
// ambiguous, and #263's AC-4 turns on that question being answerable.
|
|
1483
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS connections_one_app_per_provider_idx
|
|
1484
|
+
ON connections (tenant_id, provider)
|
|
1485
|
+
WHERE kind = 'oauth-app' AND revoked_at IS NULL`,
|
|
1486
|
+
],
|
|
1487
|
+
down: [
|
|
1488
|
+
`DROP INDEX IF EXISTS connections_one_app_per_provider_idx`,
|
|
1489
|
+
`ALTER TABLE connections DROP COLUMN IF EXISTS kind`,
|
|
1490
|
+
],
|
|
1491
|
+
},
|
|
1492
|
+
{
|
|
1493
|
+
/**
|
|
1494
|
+
* `waiting-for-connection` — the third pause, task #264.
|
|
1495
|
+
*
|
|
1496
|
+
* The CHECK constraint mirrors `RUN_STATUSES`, so a new status has to be added in **both** places or every
|
|
1497
|
+
* write of it is rejected by the database. Found by `run-events-retention.test.ts`, which iterates every
|
|
1498
|
+
* status and inserts one — a test that exists precisely so the two lists cannot drift apart silently.
|
|
1499
|
+
*
|
|
1500
|
+
* Dropped and recreated rather than altered: Postgres has no `ALTER CONSTRAINT` for a CHECK's expression,
|
|
1501
|
+
* and `IF EXISTS`/`IF NOT EXISTS` keep both statements re-runnable, which `migrate` requires.
|
|
1502
|
+
*/
|
|
1503
|
+
id: "0034_run_status_waiting_for_connection",
|
|
1504
|
+
up: [
|
|
1505
|
+
`ALTER TABLE runs DROP CONSTRAINT IF EXISTS runs_status_check`,
|
|
1506
|
+
`ALTER TABLE runs ADD CONSTRAINT runs_status_check CHECK (status IN (
|
|
1507
|
+
'queued', 'running', 'waiting-for-question', 'waiting-for-approval',
|
|
1508
|
+
'waiting-for-connection', 'retry-pending', 'completed', 'failed', 'cancelled'
|
|
1509
|
+
))`,
|
|
1510
|
+
],
|
|
1511
|
+
down: [
|
|
1512
|
+
`ALTER TABLE runs DROP CONSTRAINT IF EXISTS runs_status_check`,
|
|
1513
|
+
`ALTER TABLE runs ADD CONSTRAINT runs_status_check CHECK (status IN (
|
|
1514
|
+
'queued', 'running', 'waiting-for-question', 'waiting-for-approval',
|
|
1515
|
+
'retry-pending', 'completed', 'failed', 'cancelled'
|
|
1516
|
+
))`,
|
|
1517
|
+
],
|
|
1518
|
+
},
|
|
1519
|
+
{
|
|
1520
|
+
/**
|
|
1521
|
+
* The knowledge graph — REQ-064 (#270), task #271.
|
|
1522
|
+
*
|
|
1523
|
+
* In `MIGRATIONS` rather than beside the pgvector ones, because none of this needs the extension: entities
|
|
1524
|
+
* and edges are ordinary rows, and the embeddings they were derived from live in `knowledge_chunks`. A
|
|
1525
|
+
* deployment without pgvector can still run GraphRAG's `graph-local`, which is a real configuration.
|
|
1526
|
+
*
|
|
1527
|
+
* **Contributions are the stored truth; the merged rows are a cache.** `knowledge_graph_contributions`
|
|
1528
|
+
* records what each source asserted, and the entity and relationship tables are rebuilt from it on write.
|
|
1529
|
+
* That is what makes pruning correct with no reference counting: an entity exists exactly while some
|
|
1530
|
+
* contribution names it, and re-indexing one document withdraws only that document's claims.
|
|
1531
|
+
*/
|
|
1532
|
+
id: "0035_knowledge_graph",
|
|
1533
|
+
up: [
|
|
1534
|
+
`CREATE TABLE IF NOT EXISTS knowledge_graph_settings (
|
|
1535
|
+
tenant_id text PRIMARY KEY,
|
|
1536
|
+
enabled boolean NOT NULL DEFAULT false,
|
|
1537
|
+
updated_at timestamptz NOT NULL DEFAULT now()
|
|
1538
|
+
)`,
|
|
1539
|
+
// Stored independently of the tenant switch — a source marked while GraphRAG is off keeps its flag, so
|
|
1540
|
+
// enabling the tenant later picks it up without re-marking anything.
|
|
1541
|
+
`CREATE TABLE IF NOT EXISTS knowledge_graph_sources (
|
|
1542
|
+
tenant_id text NOT NULL,
|
|
1543
|
+
source_type text NOT NULL,
|
|
1544
|
+
source_id text NOT NULL,
|
|
1545
|
+
enabled boolean NOT NULL DEFAULT false,
|
|
1546
|
+
PRIMARY KEY (tenant_id, source_type, source_id)
|
|
1547
|
+
)`,
|
|
1548
|
+
`CREATE TABLE IF NOT EXISTS knowledge_graph_contributions (
|
|
1549
|
+
tenant_id text NOT NULL,
|
|
1550
|
+
source_type text NOT NULL,
|
|
1551
|
+
source_id text NOT NULL,
|
|
1552
|
+
entities jsonb NOT NULL,
|
|
1553
|
+
relationships jsonb NOT NULL,
|
|
1554
|
+
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
1555
|
+
PRIMARY KEY (tenant_id, source_type, source_id)
|
|
1556
|
+
)`,
|
|
1557
|
+
`CREATE TABLE IF NOT EXISTS knowledge_graph_entities (
|
|
1558
|
+
tenant_id text NOT NULL,
|
|
1559
|
+
id text NOT NULL,
|
|
1560
|
+
name text NOT NULL,
|
|
1561
|
+
type text NOT NULL,
|
|
1562
|
+
description text,
|
|
1563
|
+
surface_forms text[] NOT NULL,
|
|
1564
|
+
provenance text[] NOT NULL,
|
|
1565
|
+
PRIMARY KEY (tenant_id, id),
|
|
1566
|
+
-- Provenance is structural, so the database says so too. A graph claim with no chunk behind it is one
|
|
1567
|
+
-- the retriever would present as cited, and an application-level check is one a future writer skips.
|
|
1568
|
+
CONSTRAINT knowledge_graph_entities_provenance CHECK (cardinality(provenance) > 0)
|
|
1569
|
+
)`,
|
|
1570
|
+
`CREATE TABLE IF NOT EXISTS knowledge_graph_relationships (
|
|
1571
|
+
tenant_id text NOT NULL,
|
|
1572
|
+
id text NOT NULL,
|
|
1573
|
+
from_id text NOT NULL,
|
|
1574
|
+
to_id text NOT NULL,
|
|
1575
|
+
type text NOT NULL,
|
|
1576
|
+
description text,
|
|
1577
|
+
weight integer NOT NULL DEFAULT 1,
|
|
1578
|
+
provenance text[] NOT NULL,
|
|
1579
|
+
PRIMARY KEY (tenant_id, id),
|
|
1580
|
+
CONSTRAINT knowledge_graph_relationships_provenance CHECK (cardinality(provenance) > 0)
|
|
1581
|
+
)`,
|
|
1582
|
+
`CREATE INDEX IF NOT EXISTS knowledge_graph_entities_type_idx
|
|
1583
|
+
ON knowledge_graph_entities (tenant_id, type)`,
|
|
1584
|
+
// Traversal reads edges by either endpoint, so both directions are indexed. One index on `from_id` would
|
|
1585
|
+
// make half of every neighbourhood query a sequential scan.
|
|
1586
|
+
`CREATE INDEX IF NOT EXISTS knowledge_graph_relationships_from_idx
|
|
1587
|
+
ON knowledge_graph_relationships (tenant_id, from_id)`,
|
|
1588
|
+
`CREATE INDEX IF NOT EXISTS knowledge_graph_relationships_to_idx
|
|
1589
|
+
ON knowledge_graph_relationships (tenant_id, to_id)`,
|
|
1590
|
+
/*
|
|
1591
|
+
* Communities — #272. `fingerprint` is what the cluster contains; `summary_fingerprint` is what the
|
|
1592
|
+
* summary was written against. A community is stale exactly when they differ, which makes staleness a
|
|
1593
|
+
* comparison rather than a timestamp somebody has to interpret.
|
|
1594
|
+
*/
|
|
1595
|
+
`CREATE TABLE IF NOT EXISTS knowledge_graph_communities (
|
|
1596
|
+
tenant_id text NOT NULL,
|
|
1597
|
+
id text NOT NULL,
|
|
1598
|
+
level integer NOT NULL,
|
|
1599
|
+
entity_ids text[] NOT NULL,
|
|
1600
|
+
relationship_ids text[] NOT NULL,
|
|
1601
|
+
chunk_ids text[] NOT NULL,
|
|
1602
|
+
fingerprint text NOT NULL,
|
|
1603
|
+
summary text,
|
|
1604
|
+
summary_fingerprint text,
|
|
1605
|
+
summarised_at timestamptz,
|
|
1606
|
+
PRIMARY KEY (tenant_id, id)
|
|
1607
|
+
)`,
|
|
1608
|
+
`CREATE INDEX IF NOT EXISTS knowledge_graph_communities_level_idx
|
|
1609
|
+
ON knowledge_graph_communities (tenant_id, level, id)`,
|
|
1610
|
+
],
|
|
1611
|
+
down: [
|
|
1612
|
+
`DROP TABLE IF EXISTS knowledge_graph_communities`,
|
|
1613
|
+
`DROP TABLE IF EXISTS knowledge_graph_relationships`,
|
|
1614
|
+
`DROP TABLE IF EXISTS knowledge_graph_entities`,
|
|
1615
|
+
`DROP TABLE IF EXISTS knowledge_graph_contributions`,
|
|
1616
|
+
`DROP TABLE IF EXISTS knowledge_graph_sources`,
|
|
1617
|
+
`DROP TABLE IF EXISTS knowledge_graph_settings`,
|
|
1618
|
+
],
|
|
1619
|
+
},
|
|
1422
1620
|
];
|
|
1423
1621
|
/**
|
|
1424
1622
|
* pgvector migrations, applied separately (#135).
|
|
@@ -93,7 +93,19 @@ export const createPostgresRunStore = (sql) => {
|
|
|
93
93
|
* A live lease held by *another* worker excludes the row; the current holder re-claiming is
|
|
94
94
|
* allowed (idempotent re-entry after a transient failure).
|
|
95
95
|
*/
|
|
96
|
-
async claim({ tenantId, id, workerId, leaseMs, now }) {
|
|
96
|
+
async claim({ tenantId, id, workerId, leaseMs, now, maxConcurrent }) {
|
|
97
|
+
/**
|
|
98
|
+
* The per-tenant concurrency cap, **in the same statement as the claim** — #265.
|
|
99
|
+
*
|
|
100
|
+
* A correlated subquery rather than a count read beforehand, and that is the whole point. Reading the
|
|
101
|
+
* count and then updating is check-then-act across processes: two workers both read "3 of 4 used", both
|
|
102
|
+
* claim, and the tenant runs at 5. Inside the `WHERE`, the count is evaluated against the row versions
|
|
103
|
+
* this statement locks, so the two claims serialise and the second one's subquery sees the first.
|
|
104
|
+
*
|
|
105
|
+
* `id <> $2` keeps the run from counting against itself, so recovering an expired lease is never blocked
|
|
106
|
+
* by the limit it is trying to restore. A non-positive or absent limit is passed as NULL and the clause
|
|
107
|
+
* short-circuits to true.
|
|
108
|
+
*/
|
|
97
109
|
const rows = await sql.query(`UPDATE runs
|
|
98
110
|
SET status = 'running',
|
|
99
111
|
claimed_by = $3,
|
|
@@ -109,10 +121,22 @@ export const createPostgresRunStore = (sql) => {
|
|
|
109
121
|
OR claimed_by = $3
|
|
110
122
|
OR lease_expires_at IS NULL
|
|
111
123
|
OR lease_expires_at <= $4::timestamptz)
|
|
112
|
-
|
|
124
|
+
AND ($6::int IS NULL
|
|
125
|
+
OR (SELECT count(*) FROM runs live
|
|
126
|
+
WHERE live.tenant_id = $1
|
|
127
|
+
AND live.id <> $2
|
|
128
|
+
AND live.status = 'running'
|
|
129
|
+
AND live.lease_expires_at > $4::timestamptz) < $6::int)
|
|
130
|
+
RETURNING *`, [tenantId, id, workerId, now, String(leaseMs), maxConcurrent !== undefined && maxConcurrent > 0 ? maxConcurrent : null]);
|
|
113
131
|
const row = rows[0];
|
|
114
132
|
return row ? toRun(row) : null;
|
|
115
133
|
},
|
|
134
|
+
async countLive({ tenantId, now }) {
|
|
135
|
+
const rows = await sql.query(`SELECT count(*) AS live FROM runs
|
|
136
|
+
WHERE tenant_id = $1 AND status = 'running' AND lease_expires_at > $2::timestamptz`, [tenantId, now]);
|
|
137
|
+
// Postgres returns count() as bigint, which the driver hands back as a string.
|
|
138
|
+
return Number(rows[0]?.live ?? 0);
|
|
139
|
+
},
|
|
116
140
|
/** False when the claim was lost (reaped or stolen) so the worker aborts rather than continuing. */
|
|
117
141
|
async keepalive({ tenantId, id, workerId, leaseMs, now }) {
|
|
118
142
|
const rows = await sql.query(`UPDATE runs
|
|
@@ -3,12 +3,55 @@
|
|
|
3
3
|
*
|
|
4
4
|
* A `SchemaManager` over the reversible migrations: it provisions a fresh database on startup
|
|
5
5
|
* (`auto`), logs the diff and refuses (`plan`), or leaves the schema to managed migrations
|
|
6
|
-
* (`off`, the default for Postgres). Forward-only and idempotent — running it twice is a no-op
|
|
7
|
-
*
|
|
6
|
+
* (`off`, the default for Postgres). Forward-only and idempotent — running it twice is a no-op.
|
|
7
|
+
*
|
|
8
|
+
* **Concurrent callers do not double-provision, and one of them may still fail.** Measured for #252: two
|
|
9
|
+
* simultaneous `apply()` runs against one database leave the ledger correct (every migration recorded once,
|
|
10
|
+
* `ON CONFLICT DO NOTHING` doing its job) and crash the loser with
|
|
11
|
+
* `duplicate key value violates unique constraint "pg_type_typname_nsp_index"` — Postgres's own type catalogue,
|
|
12
|
+
* racing on DDL. The data is safe; the process is not, and the error names nothing an operator can act on.
|
|
13
|
+
*
|
|
14
|
+
* Both callers therefore take a **session advisory lock** on a single checked-out connection before applying:
|
|
15
|
+
* `retinue migrate` since #252, and `auto` mode at startup since #266. The callers serialise instead of racing.
|
|
16
|
+
*
|
|
17
|
+
* ## How the lock reaches a manager written against `SqlExecutor` — #266, AC-2
|
|
18
|
+
*
|
|
19
|
+
* It does not. `SqlExecutor` has only `query`, and a `pg_advisory_lock` taken through a pool-backed executor
|
|
20
|
+
* would be held by a connection nobody gets back — `pool.query` picks a different one per call, so the unlock
|
|
21
|
+
* would land on a different session and the lock would leak until the process exited.
|
|
22
|
+
*
|
|
23
|
+
* So `provisionSchema` takes an **optional `ConnectionOpener`** and builds the manager over the locked
|
|
24
|
+
* connection when it has one. Three options were weighed and this is why:
|
|
25
|
+
*
|
|
26
|
+
* - *Widen `SqlExecutor`* — rejected. Every adapter that will never need a session lock would have to answer
|
|
27
|
+
* for one, and PGlite genuinely cannot.
|
|
28
|
+
* - *Lock in the host's boot path* — rejected as the primary mechanism, though `boot.ts` is where the opener
|
|
29
|
+
* comes from. It puts the correctness of provisioning in the caller, so every future host has to remember,
|
|
30
|
+
* and the one that forgets fails only under concurrency.
|
|
31
|
+
* - *An optional opener here* — chosen. The knowledge stays with the thing being protected, a caller that has
|
|
32
|
+
* an opener is protected without asking, and a caller that has none behaves exactly as before.
|
|
33
|
+
*
|
|
34
|
+
* **A caller with no opener still works** (AC-3): PGlite and the in-memory paths are single-process, so there
|
|
35
|
+
* is nothing to serialise, and requiring a lock they cannot provide would be a new dependency for the one case
|
|
36
|
+
* that never needed it. The result reports which happened, so "unlocked" is visible rather than assumed.
|
|
8
37
|
*/
|
|
9
38
|
import { type Migration } from "./migrations.js";
|
|
10
39
|
import type { SqlExecutor } from "./sql.js";
|
|
40
|
+
import type { ConnectionOpener } from "./transaction.js";
|
|
11
41
|
export type SchemaMode = "auto" | "plan" | "off";
|
|
42
|
+
/**
|
|
43
|
+
* The advisory-lock key migrations serialise on — #252, and #266's AC-4.
|
|
44
|
+
*
|
|
45
|
+
* **Exported and shared, rather than agreed.** The CLI defined this itself, which meant AC-4 — "the key matches
|
|
46
|
+
* the CLI's, or the two do not serialise against each other" — was a property of two constants happening to be
|
|
47
|
+
* equal. Two constants that must be equal are one constant; a copy that drifts produces two locks and no
|
|
48
|
+
* serialisation, and the symptom is the original crash returning with the fix apparently in place.
|
|
49
|
+
*
|
|
50
|
+
* Arbitrary, and it has to be: Postgres advisory locks are a flat 64-bit namespace with no registry, so the
|
|
51
|
+
* only protection against collision is picking something nobody else would. Derived from the ASCII of
|
|
52
|
+
* "retinue" so it is reproducible rather than a magic number somebody will tidy up.
|
|
53
|
+
*/
|
|
54
|
+
export declare const MIGRATION_LOCK = 125780070329973;
|
|
12
55
|
export type SchemaChange = {
|
|
13
56
|
readonly id: string;
|
|
14
57
|
readonly statements: readonly string[];
|
|
@@ -26,6 +69,14 @@ export type ProvisionResult = {
|
|
|
26
69
|
readonly mode: SchemaMode;
|
|
27
70
|
readonly applied: readonly string[];
|
|
28
71
|
readonly planned: readonly SchemaChange[];
|
|
72
|
+
/**
|
|
73
|
+
* Whether the apply was serialised by an advisory lock — #266.
|
|
74
|
+
*
|
|
75
|
+
* Reported so "unlocked" is a visible fact rather than an assumption. `false` is correct and expected for a
|
|
76
|
+
* single-process caller; on a pool-backed deployment it means the opener was not passed, which is the
|
|
77
|
+
* configuration mistake this field exists to make findable.
|
|
78
|
+
*/
|
|
79
|
+
readonly locked: boolean;
|
|
29
80
|
};
|
|
30
81
|
/**
|
|
31
82
|
* Startup provisioning per mode. `off` (Postgres default) leaves the schema to managed
|
|
@@ -35,5 +86,13 @@ export declare const provisionSchema: (sql: SqlExecutor, options?: {
|
|
|
35
86
|
readonly mode?: SchemaMode;
|
|
36
87
|
readonly log?: (message: string) => void;
|
|
37
88
|
readonly migrations?: readonly Migration[];
|
|
89
|
+
/**
|
|
90
|
+
* Opens one connection and holds it — the primitive an advisory lock needs, and the one `SqlExecutor`
|
|
91
|
+
* deliberately does not have. See the header for why this is a parameter rather than a widened port.
|
|
92
|
+
*
|
|
93
|
+
* Optional: a single-process caller has nothing to serialise. Supplying it is what turns concurrent
|
|
94
|
+
* `auto`-mode provisioning from "one worker crashes" into "one applies, the rest find nothing to do".
|
|
95
|
+
*/
|
|
96
|
+
readonly open?: ConnectionOpener;
|
|
38
97
|
}) => Promise<ProvisionResult>;
|
|
39
98
|
//# sourceMappingURL=schema.d.ts.map
|
|
@@ -3,10 +3,52 @@
|
|
|
3
3
|
*
|
|
4
4
|
* A `SchemaManager` over the reversible migrations: it provisions a fresh database on startup
|
|
5
5
|
* (`auto`), logs the diff and refuses (`plan`), or leaves the schema to managed migrations
|
|
6
|
-
* (`off`, the default for Postgres). Forward-only and idempotent — running it twice is a no-op
|
|
7
|
-
*
|
|
6
|
+
* (`off`, the default for Postgres). Forward-only and idempotent — running it twice is a no-op.
|
|
7
|
+
*
|
|
8
|
+
* **Concurrent callers do not double-provision, and one of them may still fail.** Measured for #252: two
|
|
9
|
+
* simultaneous `apply()` runs against one database leave the ledger correct (every migration recorded once,
|
|
10
|
+
* `ON CONFLICT DO NOTHING` doing its job) and crash the loser with
|
|
11
|
+
* `duplicate key value violates unique constraint "pg_type_typname_nsp_index"` — Postgres's own type catalogue,
|
|
12
|
+
* racing on DDL. The data is safe; the process is not, and the error names nothing an operator can act on.
|
|
13
|
+
*
|
|
14
|
+
* Both callers therefore take a **session advisory lock** on a single checked-out connection before applying:
|
|
15
|
+
* `retinue migrate` since #252, and `auto` mode at startup since #266. The callers serialise instead of racing.
|
|
16
|
+
*
|
|
17
|
+
* ## How the lock reaches a manager written against `SqlExecutor` — #266, AC-2
|
|
18
|
+
*
|
|
19
|
+
* It does not. `SqlExecutor` has only `query`, and a `pg_advisory_lock` taken through a pool-backed executor
|
|
20
|
+
* would be held by a connection nobody gets back — `pool.query` picks a different one per call, so the unlock
|
|
21
|
+
* would land on a different session and the lock would leak until the process exited.
|
|
22
|
+
*
|
|
23
|
+
* So `provisionSchema` takes an **optional `ConnectionOpener`** and builds the manager over the locked
|
|
24
|
+
* connection when it has one. Three options were weighed and this is why:
|
|
25
|
+
*
|
|
26
|
+
* - *Widen `SqlExecutor`* — rejected. Every adapter that will never need a session lock would have to answer
|
|
27
|
+
* for one, and PGlite genuinely cannot.
|
|
28
|
+
* - *Lock in the host's boot path* — rejected as the primary mechanism, though `boot.ts` is where the opener
|
|
29
|
+
* comes from. It puts the correctness of provisioning in the caller, so every future host has to remember,
|
|
30
|
+
* and the one that forgets fails only under concurrency.
|
|
31
|
+
* - *An optional opener here* — chosen. The knowledge stays with the thing being protected, a caller that has
|
|
32
|
+
* an opener is protected without asking, and a caller that has none behaves exactly as before.
|
|
33
|
+
*
|
|
34
|
+
* **A caller with no opener still works** (AC-3): PGlite and the in-memory paths are single-process, so there
|
|
35
|
+
* is nothing to serialise, and requiring a lock they cannot provide would be a new dependency for the one case
|
|
36
|
+
* that never needed it. The result reports which happened, so "unlocked" is visible rather than assumed.
|
|
8
37
|
*/
|
|
9
38
|
import { MIGRATION_LEDGER, MIGRATIONS, appliedMigrationIds } from "./migrations.js";
|
|
39
|
+
/**
|
|
40
|
+
* The advisory-lock key migrations serialise on — #252, and #266's AC-4.
|
|
41
|
+
*
|
|
42
|
+
* **Exported and shared, rather than agreed.** The CLI defined this itself, which meant AC-4 — "the key matches
|
|
43
|
+
* the CLI's, or the two do not serialise against each other" — was a property of two constants happening to be
|
|
44
|
+
* equal. Two constants that must be equal are one constant; a copy that drifts produces two locks and no
|
|
45
|
+
* serialisation, and the symptom is the original crash returning with the fix apparently in place.
|
|
46
|
+
*
|
|
47
|
+
* Arbitrary, and it has to be: Postgres advisory locks are a flat 64-bit namespace with no registry, so the
|
|
48
|
+
* only protection against collision is picking something nobody else would. Derived from the ASCII of
|
|
49
|
+
* "retinue" so it is reproducible rather than a magic number somebody will tidy up.
|
|
50
|
+
*/
|
|
51
|
+
export const MIGRATION_LOCK = 0x72_65_74_69_6e_75; // "retinu"
|
|
10
52
|
export const createSchemaManager = (sql, migrations = MIGRATIONS) => {
|
|
11
53
|
// Read-only: if the tracking table doesn't exist yet, nothing has been applied. `plan()` and
|
|
12
54
|
// `currentVersion()` therefore have no side effects; only `apply()` creates the table.
|
|
@@ -48,23 +90,66 @@ export const createSchemaManager = (sql, migrations = MIGRATIONS) => {
|
|
|
48
90
|
*/
|
|
49
91
|
export const provisionSchema = async (sql, options = {}) => {
|
|
50
92
|
const mode = options.mode ?? "off";
|
|
51
|
-
const manager = createSchemaManager(sql, options.migrations);
|
|
52
93
|
const log = options.log ?? (() => { });
|
|
53
94
|
if (mode === "off") {
|
|
54
|
-
return { mode, applied: [], planned: [] };
|
|
95
|
+
return { mode, applied: [], planned: [], locked: false };
|
|
55
96
|
}
|
|
56
|
-
const planned = await manager.plan();
|
|
57
97
|
if (mode === "plan") {
|
|
98
|
+
/**
|
|
99
|
+
* Read-only, and takes **no lock** — the same decision `retinue migrate --dry-run` makes.
|
|
100
|
+
*
|
|
101
|
+
* `plan()` and `currentVersion()` are documented as side-effect free; only `apply()` creates the ledger.
|
|
102
|
+
* Taking a lock here would serialise every booting worker behind a read that changes nothing, and a dry
|
|
103
|
+
* run that waited on a migration lock would be a dry run with a production dependency.
|
|
104
|
+
*/
|
|
105
|
+
const planned = await createSchemaManager(sql, options.migrations).plan();
|
|
58
106
|
if (planned.length === 0)
|
|
59
107
|
log("schema is up to date; no changes to apply");
|
|
60
108
|
else
|
|
61
109
|
log(`schema plan — ${planned.length} pending migration(s): ${planned.map((c) => c.id).join(", ")}`);
|
|
62
|
-
return { mode, applied: [], planned };
|
|
110
|
+
return { mode, applied: [], planned, locked: false };
|
|
63
111
|
}
|
|
64
112
|
// auto
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
113
|
+
if (options.open === undefined) {
|
|
114
|
+
/**
|
|
115
|
+
* No opener: apply directly, exactly as before.
|
|
116
|
+
*
|
|
117
|
+
* Correct for PGlite and the in-memory paths, which are single-process. On a pool-backed deployment with
|
|
118
|
+
* several workers this is the path that produces the original crash, which is why `locked: false` is
|
|
119
|
+
* reported rather than left to be inferred.
|
|
120
|
+
*/
|
|
121
|
+
const manager = createSchemaManager(sql, options.migrations);
|
|
122
|
+
const planned = await manager.plan();
|
|
123
|
+
await manager.apply();
|
|
124
|
+
if (planned.length)
|
|
125
|
+
log(`provisioned schema: applied ${planned.map((c) => c.id).join(", ")}`);
|
|
126
|
+
return { mode, applied: planned.map((c) => c.id), planned, locked: false };
|
|
127
|
+
}
|
|
128
|
+
return await options.open(async (locked) => {
|
|
129
|
+
/**
|
|
130
|
+
* The lock, the plan and the apply all on **one** connection.
|
|
131
|
+
*
|
|
132
|
+
* The plan has to be read inside the lock, not outside it: a plan computed before waiting on the lock is
|
|
133
|
+
* a plan the winner has already invalidated, so the loser would apply migrations that are now recorded
|
|
134
|
+
* and race on the DDL anyway. Reading it here is what makes the loser report "nothing to apply".
|
|
135
|
+
*/
|
|
136
|
+
await locked.query("select pg_advisory_lock($1)", [MIGRATION_LOCK]);
|
|
137
|
+
try {
|
|
138
|
+
const manager = createSchemaManager(locked, options.migrations);
|
|
139
|
+
const planned = await manager.plan();
|
|
140
|
+
if (planned.length === 0) {
|
|
141
|
+
log(`schema already at ${manager.targetVersion()}; nothing to apply`);
|
|
142
|
+
return { mode, applied: [], planned, locked: true };
|
|
143
|
+
}
|
|
144
|
+
await manager.apply();
|
|
145
|
+
log(`provisioned schema: applied ${planned.map((c) => c.id).join(", ")}`);
|
|
146
|
+
return { mode, applied: planned.map((c) => c.id), planned, locked: true };
|
|
147
|
+
}
|
|
148
|
+
finally {
|
|
149
|
+
// Released explicitly, and by the session ending if the process dies mid-migration — which is the
|
|
150
|
+
// property that makes a crash during provisioning recoverable without an operator clearing a lock.
|
|
151
|
+
await locked.query("select pg_advisory_unlock($1)", [MIGRATION_LOCK]);
|
|
152
|
+
}
|
|
153
|
+
});
|
|
69
154
|
};
|
|
70
155
|
//# sourceMappingURL=schema.js.map
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The rate-limit counter in Redis — task #248, AC-2 and AC-3.
|
|
3
|
+
*
|
|
4
|
+
* The real implementation. An in-memory one exists for tests and is explicitly *not* usable in a deployment: two
|
|
5
|
+
* workers each holding their own `Map` would each allow the full rate, so a limit of 60/minute becomes 60×N.
|
|
6
|
+
*
|
|
7
|
+
* ## One script, not INCR then EXPIRE
|
|
8
|
+
*
|
|
9
|
+
* The obvious version is `INCR key` followed by `EXPIRE key ttl`. It is wrong, and not subtly: a process that
|
|
10
|
+
* dies, is killed, or loses its connection between the two leaves a key with **no TTL**, and that tenant is
|
|
11
|
+
* refused for ever — a permanent outage caused by the rate limiter, with no way to notice except a customer
|
|
12
|
+
* reporting it. Redis has no combined increment-and-expire command, so this is one `EVAL`, which Redis runs
|
|
13
|
+
* atomically.
|
|
14
|
+
*
|
|
15
|
+
* `PEXPIRE … NX` inside the script would also work and is not used: the key's TTL is a constant derived from the
|
|
16
|
+
* window, so setting it every increment is idempotent and one fewer thing to reason about.
|
|
17
|
+
*
|
|
18
|
+
* ## Why the key carries the window start
|
|
19
|
+
*
|
|
20
|
+
* `windowStartMs` truncates, so the key is a pure function of the moment — every process derives the same key for
|
|
21
|
+
* the same instant without coordinating, and an old window's key expires on its own rather than needing a sweep.
|
|
22
|
+
* It also makes the TTL safe to set unconditionally: a key is only ever written for the window it names.
|
|
23
|
+
*/
|
|
24
|
+
import type { RateLimitStore } from "../../usage/index.js";
|
|
25
|
+
/**
|
|
26
|
+
* The one command needed, structurally — the reasoning `RedisPublishClient` gives.
|
|
27
|
+
*
|
|
28
|
+
* `eval` rather than `defineCommand`/`evalsha`: this runs once per admitted run, not in a tight loop, and a
|
|
29
|
+
* script cache miss would need a fallback path that is more code than the saving is worth.
|
|
30
|
+
*/
|
|
31
|
+
export interface RedisEvalClient {
|
|
32
|
+
eval(script: string, numKeys: number, ...args: (string | number)[]): Promise<unknown>;
|
|
33
|
+
}
|
|
34
|
+
/** Namespaced, so a counter cannot collide with BullMQ's keys, the realtime channels, or another app's. */
|
|
35
|
+
export declare const rateLimitKey: (tenantId: string, windowStartMs: number) => string;
|
|
36
|
+
/**
|
|
37
|
+
* Increment and set the expiry in one atomic step, returning the count after incrementing.
|
|
38
|
+
*
|
|
39
|
+
* The TTL is the window plus a second of slack. The slack matters: a client whose clock is a few hundred
|
|
40
|
+
* milliseconds behind the server may still be admitting against a window Redis has already expired, and the
|
|
41
|
+
* result would be a counter that silently restarts mid-window — the failure being prevented, arriving by a
|
|
42
|
+
* different route.
|
|
43
|
+
*/
|
|
44
|
+
export declare const RATE_LIMIT_SCRIPT = "\nlocal count = redis.call('INCR', KEYS[1])\nredis.call('EXPIRE', KEYS[1], ARGV[1])\nreturn count\n";
|
|
45
|
+
export declare const createRedisRateLimitStore: (client: RedisEvalClient) => RateLimitStore;
|
|
46
|
+
//# sourceMappingURL=rate-limit.d.ts.map
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The rate-limit counter in Redis — task #248, AC-2 and AC-3.
|
|
3
|
+
*
|
|
4
|
+
* The real implementation. An in-memory one exists for tests and is explicitly *not* usable in a deployment: two
|
|
5
|
+
* workers each holding their own `Map` would each allow the full rate, so a limit of 60/minute becomes 60×N.
|
|
6
|
+
*
|
|
7
|
+
* ## One script, not INCR then EXPIRE
|
|
8
|
+
*
|
|
9
|
+
* The obvious version is `INCR key` followed by `EXPIRE key ttl`. It is wrong, and not subtly: a process that
|
|
10
|
+
* dies, is killed, or loses its connection between the two leaves a key with **no TTL**, and that tenant is
|
|
11
|
+
* refused for ever — a permanent outage caused by the rate limiter, with no way to notice except a customer
|
|
12
|
+
* reporting it. Redis has no combined increment-and-expire command, so this is one `EVAL`, which Redis runs
|
|
13
|
+
* atomically.
|
|
14
|
+
*
|
|
15
|
+
* `PEXPIRE … NX` inside the script would also work and is not used: the key's TTL is a constant derived from the
|
|
16
|
+
* window, so setting it every increment is idempotent and one fewer thing to reason about.
|
|
17
|
+
*
|
|
18
|
+
* ## Why the key carries the window start
|
|
19
|
+
*
|
|
20
|
+
* `windowStartMs` truncates, so the key is a pure function of the moment — every process derives the same key for
|
|
21
|
+
* the same instant without coordinating, and an old window's key expires on its own rather than needing a sweep.
|
|
22
|
+
* It also makes the TTL safe to set unconditionally: a key is only ever written for the window it names.
|
|
23
|
+
*/
|
|
24
|
+
/** Namespaced, so a counter cannot collide with BullMQ's keys, the realtime channels, or another app's. */
|
|
25
|
+
export const rateLimitKey = (tenantId, windowStartMs) => `agentkit:ratelimit:${tenantId}:${windowStartMs}`;
|
|
26
|
+
/**
|
|
27
|
+
* Increment and set the expiry in one atomic step, returning the count after incrementing.
|
|
28
|
+
*
|
|
29
|
+
* The TTL is the window plus a second of slack. The slack matters: a client whose clock is a few hundred
|
|
30
|
+
* milliseconds behind the server may still be admitting against a window Redis has already expired, and the
|
|
31
|
+
* result would be a counter that silently restarts mid-window — the failure being prevented, arriving by a
|
|
32
|
+
* different route.
|
|
33
|
+
*/
|
|
34
|
+
export const RATE_LIMIT_SCRIPT = `
|
|
35
|
+
local count = redis.call('INCR', KEYS[1])
|
|
36
|
+
redis.call('EXPIRE', KEYS[1], ARGV[1])
|
|
37
|
+
return count
|
|
38
|
+
`;
|
|
39
|
+
export const createRedisRateLimitStore = (client) => ({
|
|
40
|
+
async consume({ tenantId, windowStartMs, windowSeconds }) {
|
|
41
|
+
const count = await client.eval(RATE_LIMIT_SCRIPT, 1, rateLimitKey(tenantId, windowStartMs), windowSeconds + 1);
|
|
42
|
+
/**
|
|
43
|
+
* Coerced, and a non-numeric reply is a failure rather than a zero.
|
|
44
|
+
*
|
|
45
|
+
* `eval` is typed `unknown` because the client is structural. Treating an unexpected reply as `0` would make
|
|
46
|
+
* a broken Redis look like an empty counter and admit everything — a rate limiter that fails open silently.
|
|
47
|
+
* A throw is the right direction here: admission already refuses on a thrown error.
|
|
48
|
+
*/
|
|
49
|
+
const used = typeof count === "number" ? count : Number(count);
|
|
50
|
+
if (!Number.isFinite(used))
|
|
51
|
+
throw new Error(`rate limit: Redis returned ${JSON.stringify(count)} for INCR, which is not a count`);
|
|
52
|
+
return used;
|
|
53
|
+
},
|
|
54
|
+
});
|
|
55
|
+
//# sourceMappingURL=rate-limit.js.map
|
|
@@ -12,6 +12,11 @@
|
|
|
12
12
|
* client to the small interfaces here, so this package stays deployable without one.
|
|
13
13
|
*/
|
|
14
14
|
import type { AdapterCapability } from "../../persistence/index.js";
|
|
15
|
+
/**
|
|
16
|
+
* #261. Aliased rather than reimplemented, like every other store here: Supabase *is* Postgres, and a second
|
|
17
|
+
* implementation would make "identical results" something to prove per port rather than by identity.
|
|
18
|
+
*/
|
|
19
|
+
export { createPostgresConnectionStore as createSupabaseConnectionStore } from "../postgres/connections.js";
|
|
15
20
|
export { createPostgresConversationStore as createSupabaseConversationStore } from "../postgres/conversation-store.js";
|
|
16
21
|
export { createPostgresSessionStateStore as createSupabaseSessionStateStore, createPostgresThreadSummaryStore as createSupabaseThreadSummaryStore, } from "../postgres/session-state.js";
|
|
17
22
|
export { createPostgresConversationBindingStore as createSupabaseConversationBindingStore, createPostgresMessageStore as createSupabaseMessageStore, createPostgresAgentStore as createSupabaseAgentStore, } from "../postgres/message-store.js";
|
|
@@ -57,6 +62,7 @@ export { createPostgresArtifactExportStore as createSupabaseArtifactExportStore
|
|
|
57
62
|
export { createPostgresUsageRollupStore as createSupabaseUsageRollupStore } from "../postgres/rollups.js";
|
|
58
63
|
export { createPostgresEvaluationStore as createSupabaseEvaluationStore } from "../postgres/evaluation.js";
|
|
59
64
|
export { createPostgresKnowledgeStore as createSupabaseKnowledgeStore, createPostgresVectorIndex as createSupabaseVectorIndex, createPostgresKeywordIndex as createSupabaseKeywordIndex, } from "../postgres/knowledge.js";
|
|
65
|
+
export { createPostgresGraphStore as createSupabaseGraphStore } from "../postgres/graph.js";
|
|
60
66
|
export * from "./rls.js";
|
|
61
67
|
export * from "./realtime.js";
|
|
62
68
|
/**
|
|
@@ -85,4 +91,13 @@ export declare const SUPABASE_CAPABILITIES: readonly AdapterCapability[];
|
|
|
85
91
|
*/
|
|
86
92
|
export { createPostgresFlowDefinitionStore as createSupabaseFlowDefinitionStore } from "../postgres/flows.js";
|
|
87
93
|
export { createPostgresFlowExecutionStore as createSupabaseFlowExecutionStore } from "../postgres/flows.js";
|
|
94
|
+
/**
|
|
95
|
+
* The Supabase Vault `SecretCipher` — task #268.
|
|
96
|
+
*
|
|
97
|
+
* Exported from the Supabase adapters rather than from `connections/`, because it is Supabase-specific and
|
|
98
|
+
* needs a `SqlExecutor`: putting it in the general entry would make every consumer of the connections module
|
|
99
|
+
* import a database type it has no use for.
|
|
100
|
+
*/
|
|
101
|
+
export { createSupabaseVaultCipher, SUPABASE_VAULT, vaultRotationIsExternal } from "../../connections/vault-cipher.js";
|
|
102
|
+
export type { VaultCipherOptions } from "../../connections/vault-cipher.js";
|
|
88
103
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
// Conversation and session
|
|
2
|
+
/**
|
|
3
|
+
* #261. Aliased rather than reimplemented, like every other store here: Supabase *is* Postgres, and a second
|
|
4
|
+
* implementation would make "identical results" something to prove per port rather than by identity.
|
|
5
|
+
*/
|
|
6
|
+
export { createPostgresConnectionStore as createSupabaseConnectionStore } from "../postgres/connections.js";
|
|
2
7
|
export { createPostgresConversationStore as createSupabaseConversationStore } from "../postgres/conversation-store.js";
|
|
3
8
|
export { createPostgresSessionStateStore as createSupabaseSessionStateStore, createPostgresThreadSummaryStore as createSupabaseThreadSummaryStore, } from "../postgres/session-state.js";
|
|
4
9
|
export { createPostgresConversationBindingStore as createSupabaseConversationBindingStore, createPostgresMessageStore as createSupabaseMessageStore, createPostgresAgentStore as createSupabaseAgentStore, } from "../postgres/message-store.js";
|
|
@@ -48,6 +53,8 @@ export { createPostgresArtifactExportStore as createSupabaseArtifactExportStore
|
|
|
48
53
|
export { createPostgresUsageRollupStore as createSupabaseUsageRollupStore } from "../postgres/rollups.js";
|
|
49
54
|
export { createPostgresEvaluationStore as createSupabaseEvaluationStore } from "../postgres/evaluation.js";
|
|
50
55
|
export { createPostgresKnowledgeStore as createSupabaseKnowledgeStore, createPostgresVectorIndex as createSupabaseVectorIndex, createPostgresKeywordIndex as createSupabaseKeywordIndex, } from "../postgres/knowledge.js";
|
|
56
|
+
// #271. Aliased like the three above: the graph is ordinary Postgres rows, so Supabase needs no second one.
|
|
57
|
+
export { createPostgresGraphStore as createSupabaseGraphStore } from "../postgres/graph.js";
|
|
51
58
|
export * from "./rls.js";
|
|
52
59
|
export * from "./realtime.js";
|
|
53
60
|
/**
|
|
@@ -81,4 +88,12 @@ export const SUPABASE_CAPABILITIES = [
|
|
|
81
88
|
*/
|
|
82
89
|
export { createPostgresFlowDefinitionStore as createSupabaseFlowDefinitionStore } from "../postgres/flows.js";
|
|
83
90
|
export { createPostgresFlowExecutionStore as createSupabaseFlowExecutionStore } from "../postgres/flows.js";
|
|
91
|
+
/**
|
|
92
|
+
* The Supabase Vault `SecretCipher` — task #268.
|
|
93
|
+
*
|
|
94
|
+
* Exported from the Supabase adapters rather than from `connections/`, because it is Supabase-specific and
|
|
95
|
+
* needs a `SqlExecutor`: putting it in the general entry would make every consumer of the connections module
|
|
96
|
+
* import a database type it has no use for.
|
|
97
|
+
*/
|
|
98
|
+
export { createSupabaseVaultCipher, SUPABASE_VAULT, vaultRotationIsExternal } from "../../connections/vault-cipher.js";
|
|
84
99
|
//# sourceMappingURL=index.js.map
|