@sixb/pg 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -6
- package/dist/agents/agent-storage.d.ts +2 -1
- package/dist/agents/agent-storage.d.ts.map +1 -1
- package/dist/agents/rows.d.ts +4 -11
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/runs.d.ts +3 -2
- package/dist/agents/runs.d.ts.map +1 -1
- package/dist/auth-storage/service-accounts.d.ts.map +1 -1
- package/dist/auth-storage/shared.d.ts +0 -4
- package/dist/auth-storage/shared.d.ts.map +1 -1
- package/dist/connector-connection-storage.d.ts +9 -0
- package/dist/connector-connection-storage.d.ts.map +1 -0
- package/dist/index.d.ts +8 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3131 -732
- package/dist/index.js.map +28 -25
- package/dist/latest-run-query.d.ts.map +1 -1
- package/dist/migrations.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts +0 -2
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/outbox.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +3 -6
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/pg-action-run-storage.d.ts +3 -2
- package/dist/pg-action-run-storage.d.ts.map +1 -1
- package/dist/pg-ai-usage-storage.d.ts +16 -0
- package/dist/pg-ai-usage-storage.d.ts.map +1 -0
- package/dist/pg-execution-storage.d.ts +14 -0
- package/dist/pg-execution-storage.d.ts.map +1 -0
- package/dist/pg-pipeline-run-storage.d.ts +5 -2
- package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts +5 -2
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/pg-sync-run-storage.d.ts +5 -2
- package/dist/pg-sync-run-storage.d.ts.map +1 -1
- package/dist/pg-webhook-run-storage.d.ts +5 -2
- package/dist/pg-webhook-run-storage.d.ts.map +1 -1
- package/dist/pg-workflow-run-storage.d.ts +5 -3
- package/dist/pg-workflow-run-storage.d.ts.map +1 -1
- package/dist/run-list-query.d.ts +1 -1
- package/dist/run-list-query.d.ts.map +1 -1
- package/dist/testing.d.ts +6 -0
- package/dist/testing.d.ts.map +1 -0
- package/package.json +5 -2
- package/src/agents/agent-storage.ts +3 -2
- package/src/agents/rows.ts +12 -43
- package/src/agents/runs.ts +29 -15
- package/src/auth-storage/service-accounts.ts +14 -15
- package/src/auth-storage/shared.ts +0 -8
- package/src/connector-connection-storage.ts +760 -0
- package/src/index.ts +32 -15
- package/src/latest-run-query.ts +7 -1
- package/src/migrations/002-workflow-run-output.sql +27 -0
- package/src/migrations/003-merge-sync-runs.sql +3 -0
- package/src/migrations/004-executions.sql +140 -0
- package/src/migrations/005-workflow-executions.sql +10 -0
- package/src/migrations/006-narrow-ontology-source-root-index.sql +4 -0
- package/src/migrations/007-split-overrides.sql +153 -0
- package/src/migrations/008-action-executions.sql +7 -0
- package/src/migrations/009-agent-executions.sql +21 -0
- package/src/migrations/010-ai-usage-accounting-foundation.sql +76 -0
- package/src/migrations/011-sync-failure-record.sql +23 -0
- package/src/migrations/012-pipeline-failure-record.sql +47 -0
- package/src/migrations/013-workflow-failure-record.sql +45 -0
- package/src/migrations/014-agent-failure-record.sql +56 -0
- package/src/migrations/015-projection-failure-record.sql +20 -0
- package/src/migrations/016-webhook-run-failure-record.sql +19 -0
- package/src/migrations/017-action-failure-record.sql +65 -0
- package/src/migrations/018-ontology-outbox-failure-record.sql +29 -0
- package/src/migrations/019-webhook-delivery-failure-record.sql +27 -0
- package/src/migrations/020-drop-run-usage-projections.sql +8 -0
- package/src/migrations/021-sync-pipeline-executions.sql +50 -0
- package/src/migrations/022-projection-executions.sql +115 -0
- package/src/migrations/023-webhook-executions.sql +30 -0
- package/src/migrations/024-ontology-commit-executions.sql +12 -0
- package/src/migrations/025-connector-connections.sql +231 -0
- package/src/migrations.ts +79 -1
- package/src/ontology-storage/materialization-session.ts +25 -28
- package/src/ontology-storage/materialization-state.ts +159 -43
- package/src/ontology-storage/materialization-writer.ts +146 -75
- package/src/ontology-storage/materializations.ts +3 -1
- package/src/ontology-storage/outbox.ts +9 -2
- package/src/ontology-storage/shared.ts +10 -20
- package/src/pg-action-run-storage.ts +43 -67
- package/src/pg-ai-usage-storage.ts +290 -0
- package/src/pg-execution-storage.ts +147 -0
- package/src/pg-pipeline-run-storage.ts +107 -41
- package/src/pg-projection-run-storage.ts +159 -91
- package/src/pg-sync-run-storage.ts +102 -30
- package/src/pg-webhook-run-storage.ts +89 -57
- package/src/pg-workflow-run-storage.ts +101 -130
- package/src/run-list-query.ts +9 -4
- package/src/testing.ts +18 -0
- package/dist/pg-webhook-delivery-storage.d.ts +0 -17
- package/dist/pg-webhook-delivery-storage.d.ts.map +0 -1
- package/src/pg-webhook-delivery-storage.ts +0 -180
|
@@ -1,91 +1,126 @@
|
|
|
1
|
+
import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
|
|
2
|
+
import { assertWebhookRunExecution } from "@sixb/core/internal/webhook-run-storage-provider"
|
|
1
3
|
import type {
|
|
4
|
+
ExecutionStorage,
|
|
2
5
|
FinishWebhookRunInput,
|
|
6
|
+
GetWebhookRunByDeliveryInput,
|
|
3
7
|
ListWebhookRunsInput,
|
|
4
8
|
ListWebhookRunsResult,
|
|
9
|
+
RestartWebhookRunInput,
|
|
5
10
|
StartWebhookRunInput,
|
|
6
11
|
WebhookRunRecord,
|
|
7
12
|
WebhookRunStorage,
|
|
8
13
|
} from "@sixb/core/storage"
|
|
9
|
-
import { WebhookRunError } from "@sixb/core/storage"
|
|
14
|
+
import { canRetryWebhookRun, WEBHOOK_RUN_FAILURE_CODES, WebhookRunError } from "@sixb/core/storage"
|
|
10
15
|
import type { SqlParameter } from "./pg-client"
|
|
11
16
|
import { appendRunListFilters, hasEmptyStatuses, queryRunList } from "./run-list-query"
|
|
12
17
|
import { isUniqueViolation } from "./storage-errors"
|
|
13
18
|
import { type PgStoreClient, runPgTransaction } from "./transactions"
|
|
14
19
|
|
|
15
20
|
export class PgWebhookRunStorage implements WebhookRunStorage {
|
|
16
|
-
constructor(
|
|
21
|
+
constructor(
|
|
22
|
+
private readonly sql: PgStoreClient,
|
|
23
|
+
private readonly executions: ExecutionStorage
|
|
24
|
+
) {}
|
|
17
25
|
|
|
18
26
|
async start(input: StartWebhookRunInput): Promise<WebhookRunRecord> {
|
|
27
|
+
await assertWebhookRunExecution({
|
|
28
|
+
executions: this.executions,
|
|
29
|
+
projectId: input.projectId,
|
|
30
|
+
executionId: input.executionId,
|
|
31
|
+
runId: input.id,
|
|
32
|
+
route: input.route,
|
|
33
|
+
})
|
|
19
34
|
try {
|
|
20
35
|
const [row] = await this.sql<WebhookRunDatabaseRow[]>`
|
|
21
36
|
INSERT INTO webhook_runs (
|
|
22
|
-
project_id,
|
|
23
|
-
|
|
24
|
-
connector_id,
|
|
25
|
-
webhook_id,
|
|
26
|
-
status,
|
|
27
|
-
method,
|
|
28
|
-
route,
|
|
29
|
-
started_at
|
|
37
|
+
project_id, id, execution_id, connector_id, webhook_id, status, method, route,
|
|
38
|
+
started_at, request_body_bytes, request_body_sha256, idempotency_key
|
|
30
39
|
) VALUES (
|
|
31
|
-
${input.projectId},
|
|
32
|
-
${input.
|
|
33
|
-
${input.
|
|
34
|
-
${input.
|
|
35
|
-
${"running"},
|
|
36
|
-
${input.method},
|
|
37
|
-
${input.route},
|
|
38
|
-
${input.startedAt ?? new Date()}
|
|
40
|
+
${input.projectId}, ${input.id}, ${input.executionId}, ${input.connectorId},
|
|
41
|
+
${input.webhookId}, ${"running"}, ${input.method}, ${input.route},
|
|
42
|
+
${input.startedAt ?? new Date()}, ${input.requestBodyBytes},
|
|
43
|
+
${input.requestBodySha256}, ${input.idempotencyKey ?? null}
|
|
39
44
|
)
|
|
40
45
|
RETURNING *
|
|
41
46
|
`
|
|
42
|
-
|
|
43
47
|
return rowToWebhookRunRecord(row)
|
|
44
48
|
} catch (error) {
|
|
45
49
|
if (isUniqueViolation(error)) {
|
|
46
50
|
throw new WebhookRunError(
|
|
47
|
-
`[SixbPg] Webhook run '${input.id}' already exists for project '${input.projectId}'
|
|
51
|
+
`[SixbPg] Webhook run '${input.id}' or delivery key already exists for project '${input.projectId}'.`,
|
|
52
|
+
"duplicate_run"
|
|
48
53
|
)
|
|
49
54
|
}
|
|
50
|
-
|
|
51
55
|
throw error
|
|
52
56
|
}
|
|
53
57
|
}
|
|
54
58
|
|
|
55
|
-
async
|
|
59
|
+
async restart(input: RestartWebhookRunInput): Promise<WebhookRunRecord> {
|
|
56
60
|
return runPgTransaction(this.sql, async (tx) => {
|
|
57
61
|
const [existing] = await tx<WebhookRunDatabaseRow[]>`
|
|
58
62
|
SELECT * FROM webhook_runs
|
|
59
63
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
60
64
|
FOR UPDATE
|
|
61
65
|
`
|
|
62
|
-
|
|
63
66
|
if (!existing) {
|
|
64
67
|
throw new WebhookRunError(
|
|
65
|
-
`[SixbPg] Webhook run '${input.id}' not found for project '${input.projectId}'
|
|
68
|
+
`[SixbPg] Webhook run '${input.id}' not found for project '${input.projectId}'.`,
|
|
69
|
+
"not_found"
|
|
70
|
+
)
|
|
71
|
+
}
|
|
72
|
+
if (!canRetryWebhookRun(rowToWebhookRunRecord(existing))) {
|
|
73
|
+
throw new WebhookRunError(
|
|
74
|
+
`[SixbPg] Webhook run '${input.id}' cannot restart from status '${existing.status}'.`,
|
|
75
|
+
"invalid_transition"
|
|
76
|
+
)
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const [updated] = await tx<WebhookRunDatabaseRow[]>`
|
|
80
|
+
UPDATE webhook_runs
|
|
81
|
+
SET status = ${"running"}, started_at = ${input.startedAt ?? new Date()},
|
|
82
|
+
finished_at = ${null}, response_status = ${null}, error = ${null}
|
|
83
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"failed"}
|
|
84
|
+
RETURNING *
|
|
85
|
+
`
|
|
86
|
+
if (!updated) {
|
|
87
|
+
throw new WebhookRunError(
|
|
88
|
+
`[SixbPg] Webhook run '${input.id}' changed while restarting.`,
|
|
89
|
+
"invalid_transition"
|
|
66
90
|
)
|
|
67
91
|
}
|
|
92
|
+
return rowToWebhookRunRecord(updated)
|
|
93
|
+
})
|
|
94
|
+
}
|
|
68
95
|
|
|
96
|
+
async finish(input: FinishWebhookRunInput): Promise<WebhookRunRecord> {
|
|
97
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
98
|
+
const [existing] = await tx<WebhookRunDatabaseRow[]>`
|
|
99
|
+
SELECT * FROM webhook_runs
|
|
100
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
101
|
+
FOR UPDATE
|
|
102
|
+
`
|
|
103
|
+
if (!existing) {
|
|
104
|
+
throw new WebhookRunError(
|
|
105
|
+
`[SixbPg] Webhook run '${input.id}' not found for project '${input.projectId}'.`,
|
|
106
|
+
"not_found"
|
|
107
|
+
)
|
|
108
|
+
}
|
|
69
109
|
if (existing.status !== "running") {
|
|
70
110
|
throw new WebhookRunError(
|
|
71
|
-
`[SixbPg] Webhook run '${input.id}' for project '${input.projectId}' is already terminal
|
|
111
|
+
`[SixbPg] Webhook run '${input.id}' for project '${input.projectId}' is already terminal.`,
|
|
112
|
+
"invalid_transition"
|
|
72
113
|
)
|
|
73
114
|
}
|
|
74
115
|
|
|
75
116
|
const [updated] = await tx<WebhookRunDatabaseRow[]>`
|
|
76
117
|
UPDATE webhook_runs
|
|
77
|
-
SET
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
request_body_bytes = ${input.requestBodyBytes ?? null},
|
|
81
|
-
response_status = ${input.responseStatus ?? null},
|
|
82
|
-
idempotency_key = ${input.idempotencyKey ?? null},
|
|
83
|
-
delivery_claim_result = ${input.deliveryClaimResult ?? null},
|
|
84
|
-
error = ${input.status === "succeeded" ? null : (input.error ?? null)}
|
|
118
|
+
SET status = ${input.status}, finished_at = ${input.finishedAt ?? new Date()},
|
|
119
|
+
response_status = ${input.responseStatus ?? null},
|
|
120
|
+
error = ${input.status === "succeeded" ? null : serializeSixbFailure(input.error, WEBHOOK_RUN_FAILURE_CODES)}::text::jsonb
|
|
85
121
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
86
122
|
RETURNING *
|
|
87
123
|
`
|
|
88
|
-
|
|
89
124
|
return rowToWebhookRunRecord(updated)
|
|
90
125
|
})
|
|
91
126
|
}
|
|
@@ -95,38 +130,38 @@ export class PgWebhookRunStorage implements WebhookRunStorage {
|
|
|
95
130
|
SELECT * FROM webhook_runs
|
|
96
131
|
WHERE project_id = ${params.projectId} AND id = ${params.id}
|
|
97
132
|
`
|
|
133
|
+
return row ? rowToWebhookRunRecord(row) : null
|
|
134
|
+
}
|
|
98
135
|
|
|
136
|
+
async getByDelivery(input: GetWebhookRunByDeliveryInput): Promise<WebhookRunRecord | null> {
|
|
137
|
+
const [row] = await this.sql<WebhookRunDatabaseRow[]>`
|
|
138
|
+
SELECT * FROM webhook_runs
|
|
139
|
+
WHERE project_id = ${input.projectId}
|
|
140
|
+
AND connector_id = ${input.connectorId}
|
|
141
|
+
AND webhook_id = ${input.webhookId}
|
|
142
|
+
AND idempotency_key = ${input.idempotencyKey}
|
|
143
|
+
`
|
|
99
144
|
return row ? rowToWebhookRunRecord(row) : null
|
|
100
145
|
}
|
|
101
146
|
|
|
102
147
|
async list(input: ListWebhookRunsInput): Promise<ListWebhookRunsResult> {
|
|
103
|
-
if (hasEmptyStatuses(input)) {
|
|
104
|
-
return {
|
|
105
|
-
runs: [],
|
|
106
|
-
hasMore: false,
|
|
107
|
-
total: 0,
|
|
108
|
-
}
|
|
109
|
-
}
|
|
148
|
+
if (hasEmptyStatuses(input)) return { runs: [], hasMore: false, total: 0 }
|
|
110
149
|
|
|
111
150
|
const whereClauses = ["project_id = $1"]
|
|
112
151
|
const params: SqlParameter[] = [input.projectId]
|
|
113
152
|
let index = 2
|
|
114
|
-
|
|
115
153
|
if (input.connectorId) {
|
|
116
154
|
whereClauses.push(`connector_id = $${index++}`)
|
|
117
155
|
params.push(input.connectorId)
|
|
118
156
|
}
|
|
119
|
-
|
|
120
157
|
if (input.webhookId) {
|
|
121
158
|
whereClauses.push(`webhook_id = $${index++}`)
|
|
122
159
|
params.push(input.webhookId)
|
|
123
160
|
}
|
|
124
|
-
|
|
125
161
|
if (input.idempotencyKey) {
|
|
126
162
|
whereClauses.push(`idempotency_key = $${index++}`)
|
|
127
163
|
params.push(input.idempotencyKey)
|
|
128
164
|
}
|
|
129
|
-
|
|
130
165
|
index = appendRunListFilters(whereClauses, params, index, input)
|
|
131
166
|
|
|
132
167
|
const { rows, total, hasMore } = await queryRunList<WebhookRunDatabaseRow>({
|
|
@@ -139,12 +174,7 @@ export class PgWebhookRunStorage implements WebhookRunStorage {
|
|
|
139
174
|
limit: input.limit,
|
|
140
175
|
offset: input.offset,
|
|
141
176
|
})
|
|
142
|
-
|
|
143
|
-
return {
|
|
144
|
-
runs: rows.map(rowToWebhookRunRecord),
|
|
145
|
-
hasMore,
|
|
146
|
-
total,
|
|
147
|
-
}
|
|
177
|
+
return { runs: rows.map(rowToWebhookRunRecord), hasMore, total }
|
|
148
178
|
}
|
|
149
179
|
}
|
|
150
180
|
|
|
@@ -152,6 +182,7 @@ function rowToWebhookRunRecord(row: WebhookRunDatabaseRow): WebhookRunRecord {
|
|
|
152
182
|
return {
|
|
153
183
|
id: row.id,
|
|
154
184
|
projectId: row.project_id,
|
|
185
|
+
executionId: row.execution_id,
|
|
155
186
|
connectorId: row.connector_id,
|
|
156
187
|
webhookId: row.webhook_id,
|
|
157
188
|
status: row.status,
|
|
@@ -159,17 +190,18 @@ function rowToWebhookRunRecord(row: WebhookRunDatabaseRow): WebhookRunRecord {
|
|
|
159
190
|
route: row.route,
|
|
160
191
|
startedAt: new Date(row.started_at),
|
|
161
192
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
162
|
-
requestBodyBytes:
|
|
193
|
+
requestBodyBytes: Number(row.request_body_bytes),
|
|
194
|
+
requestBodySha256: row.request_body_sha256,
|
|
163
195
|
responseStatus: row.response_status != null ? Number(row.response_status) : undefined,
|
|
164
196
|
idempotencyKey: row.idempotency_key ?? undefined,
|
|
165
|
-
|
|
166
|
-
error: row.error ?? undefined,
|
|
197
|
+
error: row.error === null ? undefined : parseSixbFailure(row.error, WEBHOOK_RUN_FAILURE_CODES),
|
|
167
198
|
}
|
|
168
199
|
}
|
|
169
200
|
|
|
170
201
|
interface WebhookRunDatabaseRow {
|
|
171
202
|
readonly project_id: string
|
|
172
203
|
readonly id: string
|
|
204
|
+
readonly execution_id: string
|
|
173
205
|
readonly connector_id: string
|
|
174
206
|
readonly webhook_id: string
|
|
175
207
|
readonly status: WebhookRunRecord["status"]
|
|
@@ -177,9 +209,9 @@ interface WebhookRunDatabaseRow {
|
|
|
177
209
|
readonly route: string
|
|
178
210
|
readonly started_at: Date | string
|
|
179
211
|
readonly finished_at: Date | string | null
|
|
180
|
-
readonly request_body_bytes: number | string
|
|
212
|
+
readonly request_body_bytes: number | string
|
|
213
|
+
readonly request_body_sha256: string
|
|
181
214
|
readonly response_status: number | string | null
|
|
182
215
|
readonly idempotency_key: string | null
|
|
183
|
-
readonly
|
|
184
|
-
readonly error: string | null
|
|
216
|
+
readonly error: unknown | null
|
|
185
217
|
}
|
|
@@ -1,10 +1,17 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { JsonValue } from "@sixb/core"
|
|
2
|
+
import { normalizeRequesterGroupIds } from "@sixb/core/internal/auth"
|
|
3
|
+
import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
|
|
4
|
+
import {
|
|
5
|
+
assertWorkflowAgentNodeRunExecution,
|
|
6
|
+
assertWorkflowRunExecution,
|
|
7
|
+
} from "@sixb/core/internal/workflow-run-storage-provider"
|
|
2
8
|
import type { WorkflowIOSnapshot } from "@sixb/core/internal/workflows"
|
|
3
9
|
import type {
|
|
4
10
|
CancelWorkflowAgentNodeRunInput,
|
|
5
11
|
ConfirmWorkflowAgentNodeRunExecutionOwnershipInput,
|
|
6
12
|
ConfirmWorkflowRunExecutionOwnershipInput,
|
|
7
13
|
CreateWorkflowAgentNodeRunInput,
|
|
14
|
+
ExecutionStorage,
|
|
8
15
|
FinishWorkflowAgentNodeRunInput,
|
|
9
16
|
FinishWorkflowNodeRunInput,
|
|
10
17
|
FinishWorkflowRunInput,
|
|
@@ -32,7 +39,11 @@ import type {
|
|
|
32
39
|
WorkflowRunRecord,
|
|
33
40
|
WorkflowRunStorage,
|
|
34
41
|
} from "@sixb/core/storage"
|
|
35
|
-
import {
|
|
42
|
+
import {
|
|
43
|
+
AGENT_RUN_FAILURE_CODES,
|
|
44
|
+
WORKFLOW_RUN_FAILURE_CODES,
|
|
45
|
+
WorkflowRunError,
|
|
46
|
+
} from "@sixb/core/storage"
|
|
36
47
|
import { queryLatestRunsByOwnerId } from "./latest-run-query"
|
|
37
48
|
import type { SqlParameter } from "./pg-client"
|
|
38
49
|
import { appendRunListFilters, hasEmptyStatuses, queryRunList } from "./run-list-query"
|
|
@@ -43,39 +54,47 @@ export class PgWorkflowRunStorage implements WorkflowRunStorage {
|
|
|
43
54
|
readonly nodes: PgWorkflowNodeRunStorage
|
|
44
55
|
readonly agentNodes: PgWorkflowAgentNodeRunStorage
|
|
45
56
|
|
|
46
|
-
constructor(
|
|
57
|
+
constructor(
|
|
58
|
+
private readonly sql: PgStoreClient,
|
|
59
|
+
private readonly executions: ExecutionStorage
|
|
60
|
+
) {
|
|
47
61
|
this.nodes = new PgWorkflowNodeRunStorage(sql)
|
|
48
|
-
this.agentNodes = new PgWorkflowAgentNodeRunStorage(sql)
|
|
62
|
+
this.agentNodes = new PgWorkflowAgentNodeRunStorage(sql, executions)
|
|
49
63
|
}
|
|
50
64
|
|
|
51
65
|
async queue(input: QueueWorkflowRunInput): Promise<WorkflowRunRecord> {
|
|
52
66
|
const queuedAt = input.queuedAt ?? new Date()
|
|
67
|
+
await assertWorkflowRunExecution({
|
|
68
|
+
executions: this.executions,
|
|
69
|
+
projectId: input.projectId,
|
|
70
|
+
executionId: input.executionId,
|
|
71
|
+
runId: input.id,
|
|
72
|
+
workflowId: input.workflowId,
|
|
73
|
+
})
|
|
53
74
|
|
|
54
75
|
try {
|
|
55
76
|
const [row] = await this.sql<WorkflowRunDatabaseRow[]>`
|
|
56
77
|
INSERT INTO workflow_runs (
|
|
57
78
|
project_id,
|
|
58
79
|
id,
|
|
80
|
+
execution_id,
|
|
59
81
|
workflow_id,
|
|
60
82
|
status,
|
|
61
83
|
input,
|
|
62
84
|
queued_at,
|
|
63
85
|
started_at,
|
|
64
|
-
|
|
65
|
-
requested_by_principal_type,
|
|
66
|
-
requested_by_principal_id,
|
|
86
|
+
requester_group_ids,
|
|
67
87
|
attempt
|
|
68
88
|
) VALUES (
|
|
69
89
|
${input.projectId},
|
|
70
90
|
${input.id},
|
|
91
|
+
${input.executionId},
|
|
71
92
|
${input.workflowId},
|
|
72
93
|
${"queued"},
|
|
73
94
|
${serializeRecord(input.input)}::text::jsonb,
|
|
74
95
|
${queuedAt},
|
|
75
96
|
${queuedAt},
|
|
76
|
-
${
|
|
77
|
-
${input.requestedByPrincipal?.type ?? "system"},
|
|
78
|
-
${input.requestedByPrincipal?.id ?? "system"},
|
|
97
|
+
${JSON.stringify(normalizeRequesterGroupIds(input.requesterGroupIds))}::text::jsonb,
|
|
79
98
|
${0}
|
|
80
99
|
)
|
|
81
100
|
RETURNING *
|
|
@@ -85,7 +104,7 @@ export class PgWorkflowRunStorage implements WorkflowRunStorage {
|
|
|
85
104
|
} catch (error) {
|
|
86
105
|
if (isUniqueViolation(error)) {
|
|
87
106
|
throw new WorkflowRunError(
|
|
88
|
-
`[SixbPg] Workflow run '${input.id}' already
|
|
107
|
+
`[SixbPg] Workflow run '${input.id}' or execution '${input.executionId}' is already linked for project '${input.projectId}'.`
|
|
89
108
|
)
|
|
90
109
|
}
|
|
91
110
|
|
|
@@ -102,83 +121,29 @@ export class PgWorkflowRunStorage implements WorkflowRunStorage {
|
|
|
102
121
|
FOR UPDATE
|
|
103
122
|
`
|
|
104
123
|
|
|
105
|
-
if (existing) {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
`[SixbPg] Workflow run '${input.id}'
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
if (existing.workflow_id !== input.workflowId) {
|
|
113
|
-
throw new WorkflowRunError(
|
|
114
|
-
`[SixbPg] Workflow run '${input.id}' workflow '${input.workflowId}' does not match existing workflow '${existing.workflow_id}'.`
|
|
115
|
-
)
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
const [updated] = await tx<WorkflowRunDatabaseRow[]>`
|
|
119
|
-
UPDATE workflow_runs
|
|
120
|
-
SET
|
|
121
|
-
status = ${"running"},
|
|
122
|
-
input = ${serializeRecord(input.input)}::text::jsonb,
|
|
123
|
-
started_at = ${startedAt},
|
|
124
|
-
finished_at = ${null},
|
|
125
|
-
error = ${null},
|
|
126
|
-
source = COALESCE(
|
|
127
|
-
source,
|
|
128
|
-
${input.source ? JSON.stringify(input.source) : null}::text::jsonb
|
|
129
|
-
),
|
|
130
|
-
attempt = attempt + 1,
|
|
131
|
-
execution_token = ${input.execution?.token ?? null},
|
|
132
|
-
execution_queue_lease_expires_at = ${input.execution?.queueLeaseExpiresAt ?? null}
|
|
133
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
134
|
-
RETURNING *
|
|
135
|
-
`
|
|
136
|
-
|
|
137
|
-
return rowToWorkflowRunRecord(updated)
|
|
124
|
+
if (!existing || existing.status !== "queued") {
|
|
125
|
+
throw new WorkflowRunError(
|
|
126
|
+
existing
|
|
127
|
+
? `[SixbPg] Workflow run '${input.id}' cannot start from status '${existing.status}'.`
|
|
128
|
+
: `[SixbPg] Workflow run '${input.id}' not found for project '${input.projectId}'.`
|
|
129
|
+
)
|
|
138
130
|
}
|
|
139
131
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
execution_token,
|
|
154
|
-
execution_queue_lease_expires_at
|
|
155
|
-
) VALUES (
|
|
156
|
-
${input.projectId},
|
|
157
|
-
${input.id},
|
|
158
|
-
${input.workflowId},
|
|
159
|
-
${"running"},
|
|
160
|
-
${serializeRecord(input.input)}::text::jsonb,
|
|
161
|
-
${startedAt},
|
|
162
|
-
${input.source ? JSON.stringify(input.source) : null}::text::jsonb,
|
|
163
|
-
${input.requestedByPrincipal?.type ?? "system"},
|
|
164
|
-
${input.requestedByPrincipal?.id ?? "system"},
|
|
165
|
-
${1},
|
|
166
|
-
${input.execution?.token ?? null},
|
|
167
|
-
${input.execution?.queueLeaseExpiresAt ?? null}
|
|
168
|
-
)
|
|
169
|
-
RETURNING *
|
|
170
|
-
`
|
|
171
|
-
|
|
172
|
-
return rowToWorkflowRunRecord(row)
|
|
173
|
-
} catch (error) {
|
|
174
|
-
if (isUniqueViolation(error)) {
|
|
175
|
-
throw new WorkflowRunError(
|
|
176
|
-
`[SixbPg] Workflow run '${input.id}' already exists for project '${input.projectId}'.`
|
|
177
|
-
)
|
|
178
|
-
}
|
|
132
|
+
const [updated] = await tx<WorkflowRunDatabaseRow[]>`
|
|
133
|
+
UPDATE workflow_runs
|
|
134
|
+
SET
|
|
135
|
+
status = ${"running"},
|
|
136
|
+
started_at = ${startedAt},
|
|
137
|
+
finished_at = ${null},
|
|
138
|
+
error = ${null},
|
|
139
|
+
attempt = attempt + 1,
|
|
140
|
+
execution_token = ${input.execution?.token ?? null},
|
|
141
|
+
execution_queue_lease_expires_at = ${input.execution?.queueLeaseExpiresAt ?? null}
|
|
142
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
143
|
+
RETURNING *
|
|
144
|
+
`
|
|
179
145
|
|
|
180
|
-
|
|
181
|
-
}
|
|
146
|
+
return rowToWorkflowRunRecord(updated)
|
|
182
147
|
})
|
|
183
148
|
}
|
|
184
149
|
|
|
@@ -245,6 +210,7 @@ export class PgWorkflowRunStorage implements WorkflowRunStorage {
|
|
|
245
210
|
SET
|
|
246
211
|
status = ${input.status},
|
|
247
212
|
finished_at = ${input.finishedAt ?? new Date()},
|
|
213
|
+
output = ${serializeRecord(input.output)}::text::jsonb,
|
|
248
214
|
error = ${null},
|
|
249
215
|
execution_token = ${null},
|
|
250
216
|
execution_queue_lease_expires_at = ${null}
|
|
@@ -256,7 +222,8 @@ export class PgWorkflowRunStorage implements WorkflowRunStorage {
|
|
|
256
222
|
SET
|
|
257
223
|
status = ${input.status},
|
|
258
224
|
finished_at = ${input.finishedAt ?? new Date()},
|
|
259
|
-
|
|
225
|
+
output = ${null},
|
|
226
|
+
error = ${input.error === undefined ? null : serializeSixbFailure(input.error, WORKFLOW_RUN_FAILURE_CODES)}::text::jsonb,
|
|
260
227
|
execution_token = ${null},
|
|
261
228
|
execution_queue_lease_expires_at = ${null}
|
|
262
229
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
@@ -414,16 +381,43 @@ export class PgWorkflowRunStorage implements WorkflowRunStorage {
|
|
|
414
381
|
}
|
|
415
382
|
|
|
416
383
|
export class PgWorkflowAgentNodeRunStorage implements WorkflowAgentNodeRunStorage {
|
|
417
|
-
constructor(
|
|
384
|
+
constructor(
|
|
385
|
+
private readonly sql: PgStoreClient,
|
|
386
|
+
private readonly executions: ExecutionStorage
|
|
387
|
+
) {}
|
|
418
388
|
|
|
419
389
|
async create(input: CreateWorkflowAgentNodeRunInput): Promise<WorkflowAgentNodeRunRecord> {
|
|
390
|
+
const [parent] = await this.sql<{ execution_id: string }[]>`
|
|
391
|
+
SELECT workflow_runs.execution_id
|
|
392
|
+
FROM workflow_node_runs
|
|
393
|
+
JOIN workflow_runs
|
|
394
|
+
ON workflow_runs.project_id = workflow_node_runs.project_id
|
|
395
|
+
AND workflow_runs.id = workflow_node_runs.workflow_run_id
|
|
396
|
+
WHERE workflow_node_runs.project_id = ${input.projectId}
|
|
397
|
+
AND workflow_node_runs.id = ${input.nodeRunId}
|
|
398
|
+
AND workflow_node_runs.node_type = ${"agent"}
|
|
399
|
+
`
|
|
400
|
+
if (!parent) {
|
|
401
|
+
throw new WorkflowRunError(
|
|
402
|
+
`[SixbPg] Agent workflow node run '${input.nodeRunId}' was not found.`
|
|
403
|
+
)
|
|
404
|
+
}
|
|
405
|
+
await assertWorkflowAgentNodeRunExecution({
|
|
406
|
+
executions: this.executions,
|
|
407
|
+
projectId: input.projectId,
|
|
408
|
+
executionId: input.executionId,
|
|
409
|
+
nodeRunId: input.nodeRunId,
|
|
410
|
+
agentId: input.agentId,
|
|
411
|
+
workflowExecutionId: parent.execution_id,
|
|
412
|
+
})
|
|
413
|
+
|
|
420
414
|
try {
|
|
421
415
|
const [row] = await this.sql<WorkflowAgentNodeRunDatabaseRow[]>`
|
|
422
416
|
INSERT INTO workflow_agent_node_runs (
|
|
423
|
-
project_id, node_run_id, agent_id, status, prompt, attempt, created_at
|
|
417
|
+
project_id, node_run_id, execution_id, agent_id, status, prompt, attempt, created_at
|
|
424
418
|
) VALUES (
|
|
425
|
-
${input.projectId}, ${input.nodeRunId}, ${input.
|
|
426
|
-
${input.prompt}, ${0}, ${input.createdAt ?? new Date()}
|
|
419
|
+
${input.projectId}, ${input.nodeRunId}, ${input.executionId}, ${input.agentId},
|
|
420
|
+
${"queued"}, ${input.prompt}, ${0}, ${input.createdAt ?? new Date()}
|
|
427
421
|
) RETURNING *
|
|
428
422
|
`
|
|
429
423
|
return rowToWorkflowAgentNodeRunRecord(row)
|
|
@@ -443,8 +437,6 @@ export class PgWorkflowAgentNodeRunStorage implements WorkflowAgentNodeRunStorag
|
|
|
443
437
|
const [updated] = await tx<WorkflowAgentNodeRunDatabaseRow[]>`
|
|
444
438
|
UPDATE workflow_agent_node_runs SET
|
|
445
439
|
status = ${"running"},
|
|
446
|
-
execution_principal_type = ${input.executionPrincipal?.type ?? null},
|
|
447
|
-
execution_principal_id = ${input.executionPrincipal?.id ?? null},
|
|
448
440
|
model_id = ${input.modelId ?? null},
|
|
449
441
|
attempt = ${1},
|
|
450
442
|
execution_token = ${input.execution.token},
|
|
@@ -502,10 +494,9 @@ export class PgWorkflowAgentNodeRunStorage implements WorkflowAgentNodeRunStorag
|
|
|
502
494
|
status = ${input.status},
|
|
503
495
|
model_id = COALESCE(${input.modelId ?? null}, model_id),
|
|
504
496
|
finish_reason = ${input.finishReason ?? null},
|
|
505
|
-
usage = ${input.usage ? JSON.stringify(input.usage) : null}::text::jsonb,
|
|
506
497
|
trace = ${input.trace ? JSON.stringify(input.trace) : null}::text::jsonb,
|
|
507
498
|
diagnostics = ${input.diagnostics ? JSON.stringify(input.diagnostics) : null}::text::jsonb,
|
|
508
|
-
error = ${input.status === "succeeded" ? null : (input.error
|
|
499
|
+
error = ${input.status === "succeeded" || input.error === undefined ? null : serializeSixbFailure(input.error, AGENT_RUN_FAILURE_CODES)}::text::jsonb,
|
|
509
500
|
execution_token = ${null},
|
|
510
501
|
execution_queue_lease_expires_at = ${null},
|
|
511
502
|
completed_at = ${input.completedAt ?? new Date()}
|
|
@@ -536,7 +527,7 @@ export class PgWorkflowAgentNodeRunStorage implements WorkflowAgentNodeRunStorag
|
|
|
536
527
|
const [updated] = await tx<WorkflowAgentNodeRunDatabaseRow[]>`
|
|
537
528
|
UPDATE workflow_agent_node_runs SET
|
|
538
529
|
status = ${"cancelled"},
|
|
539
|
-
error = ${input.error
|
|
530
|
+
error = ${input.error === undefined ? null : serializeSixbFailure(input.error, AGENT_RUN_FAILURE_CODES)}::text::jsonb,
|
|
540
531
|
execution_token = ${null},
|
|
541
532
|
execution_queue_lease_expires_at = ${null},
|
|
542
533
|
completed_at = ${input.completedAt ?? new Date()}
|
|
@@ -696,7 +687,7 @@ export class PgWorkflowNodeRunStorage implements WorkflowNodeRunStorage {
|
|
|
696
687
|
status = ${input.status},
|
|
697
688
|
finished_at = ${input.finishedAt ?? new Date()},
|
|
698
689
|
output = ${null},
|
|
699
|
-
error = ${input.error
|
|
690
|
+
error = ${input.error === undefined ? null : serializeSixbFailure(input.error, WORKFLOW_RUN_FAILURE_CODES)}::text::jsonb
|
|
700
691
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
701
692
|
RETURNING *
|
|
702
693
|
`
|
|
@@ -822,18 +813,16 @@ function rowToWorkflowRunRecord(row: WorkflowRunDatabaseRow): WorkflowRunRecord
|
|
|
822
813
|
return {
|
|
823
814
|
id: row.id,
|
|
824
815
|
projectId: row.project_id,
|
|
816
|
+
executionId: row.execution_id,
|
|
825
817
|
workflowId: row.workflow_id,
|
|
826
818
|
status: row.status,
|
|
827
819
|
input: parseRecord(row.input),
|
|
820
|
+
output: row.output ? parseRecord(row.output) : undefined,
|
|
828
821
|
queuedAt: row.queued_at ? new Date(row.queued_at) : undefined,
|
|
829
822
|
startedAt: new Date(row.started_at),
|
|
830
823
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
831
|
-
error: row.error
|
|
832
|
-
|
|
833
|
-
requestedByPrincipal: {
|
|
834
|
-
type: row.requested_by_principal_type,
|
|
835
|
-
id: row.requested_by_principal_id,
|
|
836
|
-
},
|
|
824
|
+
error: row.error === null ? undefined : parseSixbFailure(row.error, WORKFLOW_RUN_FAILURE_CODES),
|
|
825
|
+
requesterGroupIds: parseJson(row.requester_group_ids),
|
|
837
826
|
attempt: Number(row.attempt),
|
|
838
827
|
...(row.execution_token && row.execution_queue_lease_expires_at
|
|
839
828
|
? {
|
|
@@ -846,14 +835,6 @@ function rowToWorkflowRunRecord(row: WorkflowRunDatabaseRow): WorkflowRunRecord
|
|
|
846
835
|
}
|
|
847
836
|
}
|
|
848
837
|
|
|
849
|
-
function parseSource(value: WorkflowRunSource | string | null): WorkflowRunSource | undefined {
|
|
850
|
-
if (value === null || value === undefined) {
|
|
851
|
-
return undefined
|
|
852
|
-
}
|
|
853
|
-
|
|
854
|
-
return typeof value === "string" ? (JSON.parse(value) as WorkflowRunSource) : value
|
|
855
|
-
}
|
|
856
|
-
|
|
857
838
|
function rowToWorkflowNodeRunRecord(row: WorkflowNodeRunDatabaseRow): WorkflowNodeRunRecord {
|
|
858
839
|
return {
|
|
859
840
|
id: row.id,
|
|
@@ -869,7 +850,7 @@ function rowToWorkflowNodeRunRecord(row: WorkflowNodeRunDatabaseRow): WorkflowNo
|
|
|
869
850
|
startedAt: new Date(row.started_at),
|
|
870
851
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
871
852
|
output: row.output ? parseRecord(row.output) : undefined,
|
|
872
|
-
error: row.error
|
|
853
|
+
error: row.error === null ? undefined : parseSixbFailure(row.error, WORKFLOW_RUN_FAILURE_CODES),
|
|
873
854
|
}
|
|
874
855
|
}
|
|
875
856
|
|
|
@@ -893,16 +874,16 @@ function canFinishWorkflowRun(
|
|
|
893
874
|
interface WorkflowRunDatabaseRow {
|
|
894
875
|
project_id: string
|
|
895
876
|
id: string
|
|
877
|
+
execution_id: string
|
|
896
878
|
workflow_id: string
|
|
897
879
|
status: WorkflowRunRecord["status"]
|
|
898
880
|
input: WorkflowIOSnapshot | string
|
|
881
|
+
output: WorkflowIOSnapshot | string | null
|
|
899
882
|
queued_at: Date | string | null
|
|
900
883
|
started_at: Date | string
|
|
901
884
|
finished_at: Date | string | null
|
|
902
|
-
error:
|
|
903
|
-
|
|
904
|
-
requested_by_principal_type: Principal["type"]
|
|
905
|
-
requested_by_principal_id: string
|
|
885
|
+
error: JsonValue | null
|
|
886
|
+
requester_group_ids: string[] | string
|
|
906
887
|
attempt: number | string
|
|
907
888
|
execution_token: string | null
|
|
908
889
|
execution_queue_lease_expires_at: Date | string | null
|
|
@@ -922,23 +903,21 @@ interface WorkflowNodeRunDatabaseRow {
|
|
|
922
903
|
started_at: Date | string
|
|
923
904
|
finished_at: Date | string | null
|
|
924
905
|
output: WorkflowIOSnapshot | string | null
|
|
925
|
-
error:
|
|
906
|
+
error: JsonValue | null
|
|
926
907
|
}
|
|
927
908
|
|
|
928
909
|
interface WorkflowAgentNodeRunDatabaseRow {
|
|
929
910
|
project_id: string
|
|
930
911
|
node_run_id: string
|
|
912
|
+
execution_id: string
|
|
931
913
|
agent_id: string
|
|
932
914
|
status: WorkflowAgentNodeRunRecord["status"]
|
|
933
915
|
prompt: string
|
|
934
|
-
execution_principal_type: "serviceAccount" | null
|
|
935
|
-
execution_principal_id: string | null
|
|
936
916
|
model_id: string | null
|
|
937
917
|
finish_reason: WorkflowAgentNodeRunRecord["finishReason"] | null
|
|
938
|
-
usage: WorkflowAgentNodeRunRecord["usage"] | string | null
|
|
939
918
|
trace: WorkflowAgentNodeRunRecord["trace"] | string | null
|
|
940
919
|
diagnostics: WorkflowAgentNodeRunRecord["diagnostics"] | string | null
|
|
941
|
-
error:
|
|
920
|
+
error: JsonValue | null
|
|
942
921
|
attempt: number | string
|
|
943
922
|
execution_token: string | null
|
|
944
923
|
execution_queue_lease_expires_at: Date | string | null
|
|
@@ -988,23 +967,15 @@ function rowToWorkflowAgentNodeRunRecord(
|
|
|
988
967
|
return {
|
|
989
968
|
projectId: row.project_id,
|
|
990
969
|
nodeRunId: row.node_run_id,
|
|
970
|
+
executionId: row.execution_id,
|
|
991
971
|
agentId: row.agent_id,
|
|
992
972
|
status: row.status,
|
|
993
973
|
prompt: row.prompt,
|
|
994
|
-
...(row.execution_principal_type && row.execution_principal_id
|
|
995
|
-
? {
|
|
996
|
-
executionPrincipal: {
|
|
997
|
-
type: row.execution_principal_type,
|
|
998
|
-
id: row.execution_principal_id,
|
|
999
|
-
},
|
|
1000
|
-
}
|
|
1001
|
-
: {}),
|
|
1002
974
|
...(row.model_id ? { modelId: row.model_id } : {}),
|
|
1003
975
|
...(row.finish_reason ? { finishReason: row.finish_reason } : {}),
|
|
1004
|
-
...(row.usage ? { usage: parseJson(row.usage) } : {}),
|
|
1005
976
|
...(row.trace ? { trace: parseJson(row.trace) } : {}),
|
|
1006
977
|
...(row.diagnostics ? { diagnostics: parseJson(row.diagnostics) } : {}),
|
|
1007
|
-
...(row.error ? { error: row.error } : {}),
|
|
978
|
+
...(row.error ? { error: parseSixbFailure(row.error, AGENT_RUN_FAILURE_CODES) } : {}),
|
|
1008
979
|
attempt: Number(row.attempt),
|
|
1009
980
|
...(row.execution_token && row.execution_queue_lease_expires_at
|
|
1010
981
|
? {
|