@sixb/pg 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -6
- package/dist/agents/agent-storage.d.ts +2 -1
- package/dist/agents/agent-storage.d.ts.map +1 -1
- package/dist/agents/rows.d.ts +4 -11
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/runs.d.ts +3 -2
- package/dist/agents/runs.d.ts.map +1 -1
- package/dist/auth-storage/service-accounts.d.ts.map +1 -1
- package/dist/auth-storage/shared.d.ts +0 -4
- package/dist/auth-storage/shared.d.ts.map +1 -1
- package/dist/connector-connection-storage.d.ts +9 -0
- package/dist/connector-connection-storage.d.ts.map +1 -0
- package/dist/index.d.ts +8 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3131 -732
- package/dist/index.js.map +28 -25
- package/dist/latest-run-query.d.ts.map +1 -1
- package/dist/migrations.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts +0 -2
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/outbox.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +3 -6
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/pg-action-run-storage.d.ts +3 -2
- package/dist/pg-action-run-storage.d.ts.map +1 -1
- package/dist/pg-ai-usage-storage.d.ts +16 -0
- package/dist/pg-ai-usage-storage.d.ts.map +1 -0
- package/dist/pg-execution-storage.d.ts +14 -0
- package/dist/pg-execution-storage.d.ts.map +1 -0
- package/dist/pg-pipeline-run-storage.d.ts +5 -2
- package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts +5 -2
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/pg-sync-run-storage.d.ts +5 -2
- package/dist/pg-sync-run-storage.d.ts.map +1 -1
- package/dist/pg-webhook-run-storage.d.ts +5 -2
- package/dist/pg-webhook-run-storage.d.ts.map +1 -1
- package/dist/pg-workflow-run-storage.d.ts +5 -3
- package/dist/pg-workflow-run-storage.d.ts.map +1 -1
- package/dist/run-list-query.d.ts +1 -1
- package/dist/run-list-query.d.ts.map +1 -1
- package/dist/testing.d.ts +6 -0
- package/dist/testing.d.ts.map +1 -0
- package/package.json +5 -2
- package/src/agents/agent-storage.ts +3 -2
- package/src/agents/rows.ts +12 -43
- package/src/agents/runs.ts +29 -15
- package/src/auth-storage/service-accounts.ts +14 -15
- package/src/auth-storage/shared.ts +0 -8
- package/src/connector-connection-storage.ts +760 -0
- package/src/index.ts +32 -15
- package/src/latest-run-query.ts +7 -1
- package/src/migrations/002-workflow-run-output.sql +27 -0
- package/src/migrations/003-merge-sync-runs.sql +3 -0
- package/src/migrations/004-executions.sql +140 -0
- package/src/migrations/005-workflow-executions.sql +10 -0
- package/src/migrations/006-narrow-ontology-source-root-index.sql +4 -0
- package/src/migrations/007-split-overrides.sql +153 -0
- package/src/migrations/008-action-executions.sql +7 -0
- package/src/migrations/009-agent-executions.sql +21 -0
- package/src/migrations/010-ai-usage-accounting-foundation.sql +76 -0
- package/src/migrations/011-sync-failure-record.sql +23 -0
- package/src/migrations/012-pipeline-failure-record.sql +47 -0
- package/src/migrations/013-workflow-failure-record.sql +45 -0
- package/src/migrations/014-agent-failure-record.sql +56 -0
- package/src/migrations/015-projection-failure-record.sql +20 -0
- package/src/migrations/016-webhook-run-failure-record.sql +19 -0
- package/src/migrations/017-action-failure-record.sql +65 -0
- package/src/migrations/018-ontology-outbox-failure-record.sql +29 -0
- package/src/migrations/019-webhook-delivery-failure-record.sql +27 -0
- package/src/migrations/020-drop-run-usage-projections.sql +8 -0
- package/src/migrations/021-sync-pipeline-executions.sql +50 -0
- package/src/migrations/022-projection-executions.sql +115 -0
- package/src/migrations/023-webhook-executions.sql +30 -0
- package/src/migrations/024-ontology-commit-executions.sql +12 -0
- package/src/migrations/025-connector-connections.sql +231 -0
- package/src/migrations.ts +79 -1
- package/src/ontology-storage/materialization-session.ts +25 -28
- package/src/ontology-storage/materialization-state.ts +159 -43
- package/src/ontology-storage/materialization-writer.ts +146 -75
- package/src/ontology-storage/materializations.ts +3 -1
- package/src/ontology-storage/outbox.ts +9 -2
- package/src/ontology-storage/shared.ts +10 -20
- package/src/pg-action-run-storage.ts +43 -67
- package/src/pg-ai-usage-storage.ts +290 -0
- package/src/pg-execution-storage.ts +147 -0
- package/src/pg-pipeline-run-storage.ts +107 -41
- package/src/pg-projection-run-storage.ts +159 -91
- package/src/pg-sync-run-storage.ts +102 -30
- package/src/pg-webhook-run-storage.ts +89 -57
- package/src/pg-workflow-run-storage.ts +101 -130
- package/src/run-list-query.ts +9 -4
- package/src/testing.ts +18 -0
- package/dist/pg-webhook-delivery-storage.d.ts +0 -17
- package/dist/pg-webhook-delivery-storage.d.ts.map +0 -1
- package/src/pg-webhook-delivery-storage.ts +0 -180
package/src/run-list-query.ts
CHANGED
|
@@ -12,7 +12,10 @@ type PgRunListTable =
|
|
|
12
12
|
|
|
13
13
|
/** Column projection — `sync_runs` needs the derived `checkpoint_present` flag. */
|
|
14
14
|
type PgRunListSelectList = "*" | "*, checkpoint IS NOT NULL AS checkpoint_present"
|
|
15
|
-
type StartedAtExpression =
|
|
15
|
+
type StartedAtExpression =
|
|
16
|
+
| "started_at"
|
|
17
|
+
| "COALESCE(started_at, created_at)"
|
|
18
|
+
| "COALESCE(started_at, queued_at)"
|
|
16
19
|
|
|
17
20
|
export function hasEmptyStatuses(input: { readonly statuses?: readonly unknown[] }): boolean {
|
|
18
21
|
return input.statuses !== undefined && input.statuses.length === 0
|
|
@@ -78,9 +81,11 @@ export async function queryRunList<TRow>(input: {
|
|
|
78
81
|
const orderColumn =
|
|
79
82
|
input.tableName === "agent_runs"
|
|
80
83
|
? "COALESCE(started_at, created_at)"
|
|
81
|
-
: input.tableName === "
|
|
82
|
-
? "
|
|
83
|
-
: "
|
|
84
|
+
: input.tableName === "sync_runs" || input.tableName === "pipeline_runs"
|
|
85
|
+
? "COALESCE(started_at, queued_at)"
|
|
86
|
+
: input.tableName === "workflow_agent_node_runs"
|
|
87
|
+
? "created_at"
|
|
88
|
+
: "started_at"
|
|
84
89
|
const idColumn = input.tableName === "workflow_agent_node_runs" ? "node_run_id" : "id"
|
|
85
90
|
let query = `
|
|
86
91
|
SELECT ${input.selectList ?? "*"} FROM ${input.tableName}
|
package/src/testing.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export interface PostgresStorageTestingAdapter {
|
|
2
|
+
advanceConnectorConnectionTime(durationMs: number): void
|
|
3
|
+
}
|
|
4
|
+
|
|
5
|
+
const adapters = new WeakMap<object, PostgresStorageTestingAdapter>()
|
|
6
|
+
|
|
7
|
+
export function registerPostgresStorageTestingAdapter(
|
|
8
|
+
storage: object,
|
|
9
|
+
advanceConnectorConnectionTime: (durationMs: number) => void
|
|
10
|
+
): void {
|
|
11
|
+
adapters.set(storage, { advanceConnectorConnectionTime })
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function getPostgresStorageTestingAdapter(storage: object): PostgresStorageTestingAdapter {
|
|
15
|
+
const adapter = adapters.get(storage)
|
|
16
|
+
if (!adapter) throw new Error("[SixbPg] Storage testing adapter is unavailable.")
|
|
17
|
+
return adapter
|
|
18
|
+
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import type { WebhookDeliveryClaimRecord, WebhookDeliveryKey, WebhookDeliveryRecord, WebhookDeliveryStorage } from "@sixb/core/storage";
|
|
2
|
-
import { type PgStoreClient } from "./transactions";
|
|
3
|
-
export declare class PgWebhookDeliveryStorage implements WebhookDeliveryStorage {
|
|
4
|
-
private readonly sql;
|
|
5
|
-
constructor(sql: PgStoreClient);
|
|
6
|
-
claim(input: WebhookDeliveryKey & {
|
|
7
|
-
receivedAt: string;
|
|
8
|
-
}): Promise<WebhookDeliveryClaimRecord>;
|
|
9
|
-
complete(input: WebhookDeliveryKey & {
|
|
10
|
-
completedAt: string;
|
|
11
|
-
}): Promise<WebhookDeliveryRecord>;
|
|
12
|
-
fail(input: WebhookDeliveryKey & {
|
|
13
|
-
failedAt: string;
|
|
14
|
-
error: string;
|
|
15
|
-
}): Promise<WebhookDeliveryRecord>;
|
|
16
|
-
}
|
|
17
|
-
//# sourceMappingURL=pg-webhook-delivery-storage.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"pg-webhook-delivery-storage.d.ts","sourceRoot":"","sources":["../src/pg-webhook-delivery-storage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,0BAA0B,EAE1B,kBAAkB,EAClB,qBAAqB,EACrB,sBAAsB,EACvB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,KAAK,aAAa,EAAoB,MAAM,gBAAgB,CAAA;AAErE,qBAAa,wBAAyB,YAAW,sBAAsB;IACzD,OAAO,CAAC,QAAQ,CAAC,GAAG;gBAAH,GAAG,EAAE,aAAa;IAEzC,KAAK,CACT,KAAK,EAAE,kBAAkB,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GACjD,OAAO,CAAC,0BAA0B,CAAC;IAyEhC,QAAQ,CACZ,KAAK,EAAE,kBAAkB,GAAG;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,GAClD,OAAO,CAAC,qBAAqB,CAAC;IAuB3B,IAAI,CACR,KAAK,EAAE,kBAAkB,GAAG;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAC9D,OAAO,CAAC,qBAAqB,CAAC;CAqBlC"}
|
|
@@ -1,180 +0,0 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
WebhookDeliveryClaimRecord,
|
|
3
|
-
WebhookDeliveryClaimResult,
|
|
4
|
-
WebhookDeliveryKey,
|
|
5
|
-
WebhookDeliveryRecord,
|
|
6
|
-
WebhookDeliveryStorage,
|
|
7
|
-
} from "@sixb/core/storage"
|
|
8
|
-
import { type PgStoreClient, runPgTransaction } from "./transactions"
|
|
9
|
-
|
|
10
|
-
export class PgWebhookDeliveryStorage implements WebhookDeliveryStorage {
|
|
11
|
-
constructor(private readonly sql: PgStoreClient) {}
|
|
12
|
-
|
|
13
|
-
async claim(
|
|
14
|
-
input: WebhookDeliveryKey & { receivedAt: string }
|
|
15
|
-
): Promise<WebhookDeliveryClaimRecord> {
|
|
16
|
-
return runPgTransaction(this.sql, async (tx) => {
|
|
17
|
-
// Atomic first claim: only one transaction can insert this scoped delivery key.
|
|
18
|
-
const [inserted] = await tx<WebhookDeliveryRow[]>`
|
|
19
|
-
INSERT INTO webhook_deliveries (
|
|
20
|
-
project_id,
|
|
21
|
-
connector_id,
|
|
22
|
-
webhook_id,
|
|
23
|
-
idempotency_key,
|
|
24
|
-
status,
|
|
25
|
-
received_at
|
|
26
|
-
) VALUES (
|
|
27
|
-
${input.projectId},
|
|
28
|
-
${input.connectorId},
|
|
29
|
-
${input.webhookId},
|
|
30
|
-
${input.idempotencyKey},
|
|
31
|
-
${"in_progress"},
|
|
32
|
-
${input.receivedAt}
|
|
33
|
-
)
|
|
34
|
-
ON CONFLICT (project_id, connector_id, webhook_id, idempotency_key) DO NOTHING
|
|
35
|
-
RETURNING *
|
|
36
|
-
`
|
|
37
|
-
|
|
38
|
-
if (inserted) {
|
|
39
|
-
return toWebhookDeliveryClaimRecord(inserted, "claimed")
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
const [existing] = await tx<WebhookDeliveryRow[]>`
|
|
43
|
-
SELECT * FROM webhook_deliveries
|
|
44
|
-
WHERE project_id = ${input.projectId}
|
|
45
|
-
AND connector_id = ${input.connectorId}
|
|
46
|
-
AND webhook_id = ${input.webhookId}
|
|
47
|
-
AND idempotency_key = ${input.idempotencyKey}
|
|
48
|
-
FOR UPDATE
|
|
49
|
-
`
|
|
50
|
-
|
|
51
|
-
if (existing?.status === "completed") {
|
|
52
|
-
return toWebhookDeliveryClaimRecord(existing, "duplicate")
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
if (existing?.status === "in_progress") {
|
|
56
|
-
return toWebhookDeliveryClaimRecord(existing, "in_progress")
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
if (existing?.status !== "failed") {
|
|
60
|
-
throw new Error("[SixbPg] Webhook delivery row disappeared during claim.")
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
// Failed deliveries are intentionally claimable so provider retries can run again.
|
|
64
|
-
const [updated] = await tx<WebhookDeliveryRow[]>`
|
|
65
|
-
UPDATE webhook_deliveries
|
|
66
|
-
SET status = ${"in_progress"},
|
|
67
|
-
received_at = ${input.receivedAt},
|
|
68
|
-
completed_at = ${null},
|
|
69
|
-
failed_at = ${null},
|
|
70
|
-
error = ${null}
|
|
71
|
-
WHERE project_id = ${input.projectId}
|
|
72
|
-
AND connector_id = ${input.connectorId}
|
|
73
|
-
AND webhook_id = ${input.webhookId}
|
|
74
|
-
AND idempotency_key = ${input.idempotencyKey}
|
|
75
|
-
RETURNING *
|
|
76
|
-
`
|
|
77
|
-
|
|
78
|
-
if (!updated) {
|
|
79
|
-
throw new Error(
|
|
80
|
-
`[SixbPg] Missing webhook delivery row after claim for ${serializeKey(input)}.`
|
|
81
|
-
)
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
return toWebhookDeliveryClaimRecord(updated, "claimed")
|
|
85
|
-
})
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
async complete(
|
|
89
|
-
input: WebhookDeliveryKey & { completedAt: string }
|
|
90
|
-
): Promise<WebhookDeliveryRecord> {
|
|
91
|
-
const [updated] = await this.sql<WebhookDeliveryRow[]>`
|
|
92
|
-
UPDATE webhook_deliveries
|
|
93
|
-
SET status = ${"completed"},
|
|
94
|
-
completed_at = ${input.completedAt},
|
|
95
|
-
failed_at = ${null},
|
|
96
|
-
error = ${null}
|
|
97
|
-
WHERE project_id = ${input.projectId}
|
|
98
|
-
AND connector_id = ${input.connectorId}
|
|
99
|
-
AND webhook_id = ${input.webhookId}
|
|
100
|
-
AND idempotency_key = ${input.idempotencyKey}
|
|
101
|
-
RETURNING *
|
|
102
|
-
`
|
|
103
|
-
|
|
104
|
-
if (!updated) {
|
|
105
|
-
throw new Error(
|
|
106
|
-
`[SixbPg] Missing webhook delivery row after complete for ${serializeKey(input)}.`
|
|
107
|
-
)
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
return toWebhookDeliveryRecord(updated)
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
async fail(
|
|
114
|
-
input: WebhookDeliveryKey & { failedAt: string; error: string }
|
|
115
|
-
): Promise<WebhookDeliveryRecord> {
|
|
116
|
-
const [updated] = await this.sql<WebhookDeliveryRow[]>`
|
|
117
|
-
UPDATE webhook_deliveries
|
|
118
|
-
SET status = ${"failed"},
|
|
119
|
-
failed_at = ${input.failedAt},
|
|
120
|
-
error = ${input.error}
|
|
121
|
-
WHERE project_id = ${input.projectId}
|
|
122
|
-
AND connector_id = ${input.connectorId}
|
|
123
|
-
AND webhook_id = ${input.webhookId}
|
|
124
|
-
AND idempotency_key = ${input.idempotencyKey}
|
|
125
|
-
RETURNING *
|
|
126
|
-
`
|
|
127
|
-
|
|
128
|
-
if (!updated) {
|
|
129
|
-
throw new Error(
|
|
130
|
-
`[SixbPg] Missing webhook delivery row after fail for ${serializeKey(input)}.`
|
|
131
|
-
)
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
return toWebhookDeliveryRecord(updated)
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
function toWebhookDeliveryClaimRecord(
|
|
139
|
-
row: WebhookDeliveryRow,
|
|
140
|
-
claimResult: WebhookDeliveryClaimResult
|
|
141
|
-
): WebhookDeliveryClaimRecord {
|
|
142
|
-
return {
|
|
143
|
-
...toWebhookDeliveryRecord(row),
|
|
144
|
-
claimResult,
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
function toWebhookDeliveryRecord(row: WebhookDeliveryRow): WebhookDeliveryRecord {
|
|
149
|
-
return {
|
|
150
|
-
projectId: row.project_id,
|
|
151
|
-
connectorId: row.connector_id,
|
|
152
|
-
webhookId: row.webhook_id,
|
|
153
|
-
idempotencyKey: row.idempotency_key,
|
|
154
|
-
status: row.status,
|
|
155
|
-
receivedAt: toIsoString(row.received_at),
|
|
156
|
-
completedAt: row.completed_at ? toIsoString(row.completed_at) : undefined,
|
|
157
|
-
failedAt: row.failed_at ? toIsoString(row.failed_at) : undefined,
|
|
158
|
-
error: row.error ?? undefined,
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
function toIsoString(value: Date | string): string {
|
|
163
|
-
return value instanceof Date ? value.toISOString() : new Date(value).toISOString()
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
function serializeKey(key: WebhookDeliveryKey): string {
|
|
167
|
-
return JSON.stringify([key.projectId, key.connectorId, key.webhookId, key.idempotencyKey])
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
interface WebhookDeliveryRow {
|
|
171
|
-
readonly project_id: string
|
|
172
|
-
readonly connector_id: string
|
|
173
|
-
readonly webhook_id: string
|
|
174
|
-
readonly idempotency_key: string
|
|
175
|
-
readonly status: WebhookDeliveryRecord["status"]
|
|
176
|
-
readonly received_at: Date | string
|
|
177
|
-
readonly completed_at: Date | string | null
|
|
178
|
-
readonly failed_at: Date | string | null
|
|
179
|
-
readonly error: string | null
|
|
180
|
-
}
|