@sixb/pg 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-storage/auth-storage.d.ts +2 -1
- package/dist/auth-storage/auth-storage.d.ts.map +1 -1
- package/dist/auth-storage/identities.d.ts +4 -0
- package/dist/auth-storage/identities.d.ts.map +1 -1
- package/dist/auth-storage/rows.d.ts +2 -1
- package/dist/auth-storage/rows.d.ts.map +1 -1
- package/dist/auth-storage/users.d.ts.map +1 -1
- package/dist/connector-connection-storage.d.ts.map +1 -1
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1807 -279
- package/dist/index.js.map +25 -21
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/storage.d.ts +10 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +5 -0
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +14 -0
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/pg-artifact-storage.d.ts +14 -2
- package/dist/pg-artifact-storage.d.ts.map +1 -1
- package/dist/pg-dataset-ingest-storage.d.ts +6 -0
- package/dist/pg-dataset-ingest-storage.d.ts.map +1 -0
- package/dist/pg-lake-retention-run-storage.d.ts +11 -0
- package/dist/pg-lake-retention-run-storage.d.ts.map +1 -0
- package/dist/pg-maintenance-heartbeat-storage.d.ts +11 -0
- package/dist/pg-maintenance-heartbeat-storage.d.ts.map +1 -0
- package/dist/pg-pipeline-run-storage.d.ts +5 -2
- package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts +5 -1
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/pg-sync-run-storage.d.ts +6 -2
- package/dist/pg-sync-run-storage.d.ts.map +1 -1
- package/dist/pg-webhook-run-storage.d.ts +2 -1
- package/dist/pg-webhook-run-storage.d.ts.map +1 -1
- package/dist/pg-workflow-run-storage.d.ts +4 -3
- package/dist/pg-workflow-run-storage.d.ts.map +1 -1
- package/dist/run-claim-sql.d.ts +37 -0
- package/dist/run-claim-sql.d.ts.map +1 -0
- package/dist/share-session-storage.d.ts +2 -1
- package/dist/share-session-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/auth-storage/auth-storage.ts +66 -0
- package/src/auth-storage/identities.ts +15 -0
- package/src/auth-storage/rows.ts +3 -0
- package/src/auth-storage/users.ts +3 -0
- package/src/connector-connection-storage.ts +33 -16
- package/src/index.ts +15 -0
- package/src/migrations/089-maintenance-heartbeats.sql +10 -0
- package/src/migrations/090-lake-retention-runs.sql +8 -0
- package/src/migrations/091-dataset-ingest-inbox.sql +38 -0
- package/src/migrations/092-run-recovery.sql +23 -0
- package/src/migrations/093-webhook-and-projection-leases.sql +10 -0
- package/src/migrations/094-artifact-board-previews.sql +14 -0
- package/src/migrations/095-artifact-open-versions.sql +101 -0
- package/src/migrations/096-artifact-documents.sql +11 -0
- package/src/migrations/097-artifact-frame-files.sql +8 -0
- package/src/migrations/098-artifact-comments.sql +36 -0
- package/src/migrations/099-object-artifact-refs.sql +12 -0
- package/src/migrations/100-shared-link-guests.sql +17 -0
- package/src/migrations/101-connector-connection-owners.sql +38 -0
- package/src/migrations/102-user-suspension-source.sql +12 -0
- package/src/migrations.ts +40 -0
- package/src/objects/storage.ts +49 -0
- package/src/ontology-storage/materialization-session.ts +4 -0
- package/src/ontology-storage/materialization-writer.ts +68 -1
- package/src/ontology-storage/materializations.ts +3 -1
- package/src/pg-artifact-storage.ts +798 -137
- package/src/pg-dataset-ingest-storage.ts +305 -0
- package/src/pg-lake-retention-run-storage.ts +40 -0
- package/src/pg-maintenance-heartbeat-storage.ts +53 -0
- package/src/pg-pipeline-run-storage.ts +85 -42
- package/src/pg-projection-run-storage.ts +61 -2
- package/src/pg-sync-run-storage.ts +100 -37
- package/src/pg-webhook-run-storage.ts +37 -11
- package/src/pg-workflow-run-storage.ts +39 -100
- package/src/run-claim-sql.ts +173 -0
- package/src/share-session-storage.ts +20 -0
package/dist/index.js
CHANGED
|
@@ -2424,6 +2424,7 @@ function rowToUserRecord(row) {
|
|
|
2424
2424
|
displayName: row.display_name ?? undefined,
|
|
2425
2425
|
avatarUrl: row.avatar_url ?? undefined,
|
|
2426
2426
|
status: row.status,
|
|
2427
|
+
...row.suspension_source === null ? {} : { suspensionSource: row.suspension_source },
|
|
2427
2428
|
createdAt: toDate(row.created_at),
|
|
2428
2429
|
updatedAt: toDate(row.updated_at)
|
|
2429
2430
|
};
|
|
@@ -3395,6 +3396,16 @@ class PgAuthUserIdentityStore {
|
|
|
3395
3396
|
`;
|
|
3396
3397
|
return rows.map(rowToIdentityRecord);
|
|
3397
3398
|
}
|
|
3399
|
+
async listForStrategy(params) {
|
|
3400
|
+
const rows = await this.sql`
|
|
3401
|
+
SELECT *
|
|
3402
|
+
FROM auth_user_identities
|
|
3403
|
+
WHERE project_id = ${params.projectId}
|
|
3404
|
+
AND strategy_id = ${params.strategyId}
|
|
3405
|
+
ORDER BY subject COLLATE "C" ASC
|
|
3406
|
+
`;
|
|
3407
|
+
return rows.map(rowToIdentityRecord);
|
|
3408
|
+
}
|
|
3398
3409
|
}
|
|
3399
3410
|
|
|
3400
3411
|
// src/auth-storage/invitations.ts
|
|
@@ -4117,6 +4128,7 @@ class PgAuthUserStore {
|
|
|
4117
4128
|
display_name,
|
|
4118
4129
|
avatar_url,
|
|
4119
4130
|
status,
|
|
4131
|
+
suspension_source,
|
|
4120
4132
|
created_at,
|
|
4121
4133
|
updated_at
|
|
4122
4134
|
) VALUES (
|
|
@@ -4126,6 +4138,7 @@ class PgAuthUserStore {
|
|
|
4126
4138
|
${input.displayName ?? null},
|
|
4127
4139
|
${input.avatarUrl ?? null},
|
|
4128
4140
|
${input.status ?? "active"},
|
|
4141
|
+
${input.status === "suspended" ? "manual" : null},
|
|
4129
4142
|
${createdAt},
|
|
4130
4143
|
${updatedAt}
|
|
4131
4144
|
)
|
|
@@ -4201,6 +4214,7 @@ class PgAuthUserStore {
|
|
|
4201
4214
|
const [row] = await tx`
|
|
4202
4215
|
UPDATE auth_users
|
|
4203
4216
|
SET status = ${input.status},
|
|
4217
|
+
suspension_source = ${input.status === "suspended" ? "manual" : null},
|
|
4204
4218
|
updated_at = ${updatedAt}
|
|
4205
4219
|
WHERE project_id = ${input.projectId}
|
|
4206
4220
|
AND id = ${input.id}
|
|
@@ -4497,6 +4511,7 @@ class PgAuthStorage {
|
|
|
4497
4511
|
display_name: user.displayName ?? null,
|
|
4498
4512
|
avatar_url: user.avatarUrl ?? null,
|
|
4499
4513
|
status: user.status,
|
|
4514
|
+
suspension_source: null,
|
|
4500
4515
|
created_at: user.createdAt,
|
|
4501
4516
|
updated_at: user.updatedAt
|
|
4502
4517
|
};
|
|
@@ -4565,6 +4580,7 @@ class PgAuthStorage {
|
|
|
4565
4580
|
const [updated] = await tx`
|
|
4566
4581
|
UPDATE auth_users
|
|
4567
4582
|
SET status = 'suspended',
|
|
4583
|
+
suspension_source = 'manual',
|
|
4568
4584
|
updated_at = ${input.suspendedAt}
|
|
4569
4585
|
WHERE project_id = ${input.projectId}
|
|
4570
4586
|
AND id = ${input.userId}
|
|
@@ -4578,6 +4594,58 @@ class PgAuthStorage {
|
|
|
4578
4594
|
return { user: rowToUserRecord(updated), previousUser: rowToUserRecord(existing) };
|
|
4579
4595
|
});
|
|
4580
4596
|
}
|
|
4597
|
+
async applyDirectoryVerdict(input) {
|
|
4598
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
4599
|
+
await lockAdvisoryKeys(tx, [authLockKey("sessions", input.projectId, input.userId)]);
|
|
4600
|
+
const existing = await getUserRowById(tx, { projectId: input.projectId, id: input.userId }, { forUpdate: true });
|
|
4601
|
+
if (!existing) {
|
|
4602
|
+
throw new AuthStorageError11("missing_user", `[Sixb] User '${input.userId}' not found for project '${input.projectId}'.`);
|
|
4603
|
+
}
|
|
4604
|
+
const previousUser = rowToUserRecord(existing);
|
|
4605
|
+
if (existing.status === "suspended" && existing.suspension_source !== "directory") {
|
|
4606
|
+
return { user: previousUser, previousUser };
|
|
4607
|
+
}
|
|
4608
|
+
if (input.verdict.status === "inactive") {
|
|
4609
|
+
if (existing.status === "suspended")
|
|
4610
|
+
return { user: previousUser, previousUser };
|
|
4611
|
+
const [suspended] = await tx`
|
|
4612
|
+
UPDATE auth_users
|
|
4613
|
+
SET status = 'suspended',
|
|
4614
|
+
suspension_source = 'directory',
|
|
4615
|
+
updated_at = ${input.appliedAt}
|
|
4616
|
+
WHERE project_id = ${input.projectId}
|
|
4617
|
+
AND id = ${input.userId}
|
|
4618
|
+
RETURNING *
|
|
4619
|
+
`;
|
|
4620
|
+
await revokeActiveSessionsForUser(tx, {
|
|
4621
|
+
projectId: input.projectId,
|
|
4622
|
+
userId: input.userId,
|
|
4623
|
+
revokedAt: input.appliedAt
|
|
4624
|
+
});
|
|
4625
|
+
return { user: rowToUserRecord(suspended), previousUser };
|
|
4626
|
+
}
|
|
4627
|
+
let row = existing;
|
|
4628
|
+
if (existing.status === "suspended") {
|
|
4629
|
+
[row] = await tx`
|
|
4630
|
+
UPDATE auth_users
|
|
4631
|
+
SET status = ${"active"},
|
|
4632
|
+
suspension_source = NULL,
|
|
4633
|
+
updated_at = ${input.appliedAt}
|
|
4634
|
+
WHERE project_id = ${input.projectId}
|
|
4635
|
+
AND id = ${input.userId}
|
|
4636
|
+
RETURNING *
|
|
4637
|
+
`;
|
|
4638
|
+
}
|
|
4639
|
+
await this.syncDirectoryGroups(tx, {
|
|
4640
|
+
completedAt: input.appliedAt,
|
|
4641
|
+
existing: [],
|
|
4642
|
+
groupIds: input.verdict.groupIds,
|
|
4643
|
+
projectId: input.projectId,
|
|
4644
|
+
userId: input.userId
|
|
4645
|
+
});
|
|
4646
|
+
return { user: rowToUserRecord(row), previousUser };
|
|
4647
|
+
});
|
|
4648
|
+
}
|
|
4581
4649
|
assertMagicLinkUsable(projectId, id, tokenHash, completedAt, row) {
|
|
4582
4650
|
if (!row) {
|
|
4583
4651
|
throw new AuthStorageError11("missing_magic_link", `[Sixb] Magic link '${id}' not found for project '${projectId}'.`);
|
|
@@ -4761,7 +4829,9 @@ function assertCompletableDeviceAuthorization(authorization, input) {
|
|
|
4761
4829
|
}
|
|
4762
4830
|
// src/connector-connection-storage.ts
|
|
4763
4831
|
import {
|
|
4764
|
-
|
|
4832
|
+
connectorConnectionOwnerFromColumns,
|
|
4833
|
+
DurableConnectorConnectionStorage,
|
|
4834
|
+
connectorConnectionOwnerColumns as ownerColumns
|
|
4765
4835
|
} from "@sixb/core/internal/connector-connection-storage-provider";
|
|
4766
4836
|
class PgConnectorConnectionStorage extends DurableConnectorConnectionStorage {
|
|
4767
4837
|
clock;
|
|
@@ -4817,15 +4887,17 @@ class PgConnectorConnectionPersistence {
|
|
|
4817
4887
|
return new Date(new Date(row.now).getTime() + this.clock.offsetMs);
|
|
4818
4888
|
}
|
|
4819
4889
|
async insertAuthorizationAttempt(record) {
|
|
4890
|
+
const owner = ownerColumns(record.owner);
|
|
4820
4891
|
try {
|
|
4821
4892
|
await this.sql`
|
|
4822
4893
|
INSERT INTO connector_authorization_attempts (
|
|
4823
|
-
project_id, connector_id, id, slot, initiated_by_execution_id,
|
|
4894
|
+
project_id, connector_id, id, owner_type, owner_id, slot, initiated_by_execution_id,
|
|
4824
4895
|
state_hash, code_verifier, redirect_uri, connection_run_id, return_to,
|
|
4825
4896
|
callback_binding_hash, reauthorization_id, reauthorization_revision,
|
|
4826
4897
|
reauthorization_connection_ids, created_at, expires_at
|
|
4827
4898
|
) VALUES (
|
|
4828
|
-
${record.projectId}, ${record.connectorId}, ${record.id},
|
|
4899
|
+
${record.projectId}, ${record.connectorId}, ${record.id},
|
|
4900
|
+
${owner.ownerType}, ${owner.ownerId}, ${record.slot},
|
|
4829
4901
|
${record.initiatedByExecutionId}, ${record.stateHash},
|
|
4830
4902
|
${record.codeVerifier === undefined ? null : jsonParameter(this.sql, record.codeVerifier)}, ${record.redirectUri},
|
|
4831
4903
|
${record.connectionRunId ?? null}, ${record.returnTo ?? null},
|
|
@@ -4887,13 +4959,14 @@ class PgConnectorConnectionPersistence {
|
|
|
4887
4959
|
const values = runValues(record);
|
|
4888
4960
|
await this.sql`
|
|
4889
4961
|
INSERT INTO connector_connection_runs (
|
|
4890
|
-
project_id, connector_id, id, kind, slot, initiated_by_execution_id,
|
|
4962
|
+
project_id, connector_id, id, kind, owner_type, owner_id, slot, initiated_by_execution_id,
|
|
4891
4963
|
status, waiting_for, processing_id, callback_started_at,
|
|
4892
4964
|
expires_at, authorization_id, cleanup_authorization_id, connections, error,
|
|
4893
4965
|
created_at, updated_at, finished_at
|
|
4894
4966
|
) VALUES (
|
|
4895
4967
|
${record.projectId}, ${record.connectorId}, ${record.id}, ${values.kind},
|
|
4896
|
-
${values.
|
|
4968
|
+
${values.ownerType}, ${values.ownerId}, ${values.slot},
|
|
4969
|
+
${values.initiatedByExecutionId}, ${values.status},
|
|
4897
4970
|
${values.waitingFor}, ${values.processingId},
|
|
4898
4971
|
${values.callbackStartedAt}, ${values.expiresAt}, ${values.authorizationId},
|
|
4899
4972
|
${values.cleanupAuthorizationId},
|
|
@@ -4927,8 +5000,8 @@ class PgConnectorConnectionPersistence {
|
|
|
4927
5000
|
const values = runValues(record);
|
|
4928
5001
|
const rows = await this.sql`
|
|
4929
5002
|
UPDATE connector_connection_runs
|
|
4930
|
-
SET kind = ${values.kind},
|
|
4931
|
-
initiated_by_execution_id = ${values.initiatedByExecutionId}, status = ${values.status},
|
|
5003
|
+
SET kind = ${values.kind}, owner_type = ${values.ownerType}, owner_id = ${values.ownerId},
|
|
5004
|
+
slot = ${values.slot}, initiated_by_execution_id = ${values.initiatedByExecutionId}, status = ${values.status},
|
|
4932
5005
|
waiting_for = ${values.waitingFor},
|
|
4933
5006
|
processing_id = ${values.processingId}, callback_started_at = ${values.callbackStartedAt},
|
|
4934
5007
|
expires_at = ${values.expiresAt}, authorization_id = ${values.authorizationId},
|
|
@@ -5005,14 +5078,15 @@ class PgConnectorConnectionPersistence {
|
|
|
5005
5078
|
assertChanged(rows, "connector authorization", record.id);
|
|
5006
5079
|
}
|
|
5007
5080
|
async insertConnection(record) {
|
|
5081
|
+
const owner = ownerColumns(record.owner);
|
|
5008
5082
|
try {
|
|
5009
5083
|
await this.sql`
|
|
5010
5084
|
INSERT INTO connector_connections (
|
|
5011
|
-
project_id, connector_id, id, authorization_id, slot, account,
|
|
5085
|
+
project_id, connector_id, id, authorization_id, owner_type, owner_id, slot, account,
|
|
5012
5086
|
account_id, status, disconnected_at, created_at, updated_at
|
|
5013
5087
|
) VALUES (
|
|
5014
5088
|
${record.projectId}, ${record.connectorId}, ${record.id}, ${record.authorizationId},
|
|
5015
|
-
${record.slot}, ${jsonParameter(this.sql, record.account)},
|
|
5089
|
+
${owner.ownerType}, ${owner.ownerId}, ${record.slot}, ${jsonParameter(this.sql, record.account)},
|
|
5016
5090
|
${record.account.id}, ${record.status}, ${record.disconnectedAt ?? null},
|
|
5017
5091
|
${record.createdAt}, ${record.updatedAt}
|
|
5018
5092
|
)
|
|
@@ -5031,9 +5105,11 @@ class PgConnectorConnectionPersistence {
|
|
|
5031
5105
|
async getConnectionBySelector(input) {
|
|
5032
5106
|
if (!this.scope.connectorId)
|
|
5033
5107
|
return null;
|
|
5108
|
+
const owner = ownerColumns(input.owner);
|
|
5034
5109
|
const rows = await this.sql`
|
|
5035
5110
|
SELECT * FROM connector_connections
|
|
5036
5111
|
WHERE project_id = ${this.scope.projectId} AND connector_id = ${this.scope.connectorId}
|
|
5112
|
+
AND owner_type = ${owner.ownerType} AND owner_id = ${owner.ownerId}
|
|
5037
5113
|
AND slot = ${input.slot}
|
|
5038
5114
|
${this.lockClause()}
|
|
5039
5115
|
`;
|
|
@@ -5069,9 +5145,11 @@ class PgConnectorConnectionPersistence {
|
|
|
5069
5145
|
return rows.map(connectionFromRow);
|
|
5070
5146
|
}
|
|
5071
5147
|
async updateConnection(record) {
|
|
5148
|
+
const owner = ownerColumns(record.owner);
|
|
5072
5149
|
const rows = await this.sql`
|
|
5073
5150
|
UPDATE connector_connections
|
|
5074
|
-
SET authorization_id = ${record.authorizationId},
|
|
5151
|
+
SET authorization_id = ${record.authorizationId}, owner_type = ${owner.ownerType},
|
|
5152
|
+
owner_id = ${owner.ownerId}, slot = ${record.slot},
|
|
5075
5153
|
account = ${jsonParameter(this.sql, record.account)},
|
|
5076
5154
|
account_id = ${record.account.id}, status = ${record.status},
|
|
5077
5155
|
disconnected_at = ${record.disconnectedAt ?? null}, updated_at = ${record.updatedAt}
|
|
@@ -5126,7 +5204,7 @@ function authorizationAttemptFromRow(row) {
|
|
|
5126
5204
|
id: row.id,
|
|
5127
5205
|
projectId: row.project_id,
|
|
5128
5206
|
connectorId: row.connector_id,
|
|
5129
|
-
owner:
|
|
5207
|
+
owner: connectorConnectionOwnerFromColumns(row.owner_type, row.owner_id),
|
|
5130
5208
|
slot: row.slot,
|
|
5131
5209
|
initiatedByExecutionId: row.initiated_by_execution_id,
|
|
5132
5210
|
stateHash: row.state_hash,
|
|
@@ -5145,6 +5223,7 @@ function authorizationAttemptFromRow(row) {
|
|
|
5145
5223
|
function runValues(record) {
|
|
5146
5224
|
return {
|
|
5147
5225
|
kind: record.kind,
|
|
5226
|
+
...ownerColumns(record.owner),
|
|
5148
5227
|
slot: record.slot,
|
|
5149
5228
|
initiatedByExecutionId: record.initiatedByExecutionId,
|
|
5150
5229
|
status: record.status,
|
|
@@ -5165,7 +5244,7 @@ function connectionRunFromRow(row) {
|
|
|
5165
5244
|
projectId: row.project_id,
|
|
5166
5245
|
connectorId: row.connector_id,
|
|
5167
5246
|
kind: row.kind,
|
|
5168
|
-
owner:
|
|
5247
|
+
owner: connectorConnectionOwnerFromColumns(row.owner_type, row.owner_id),
|
|
5169
5248
|
slot: row.slot,
|
|
5170
5249
|
initiatedByExecutionId: row.initiated_by_execution_id,
|
|
5171
5250
|
createdAt: date(row.created_at),
|
|
@@ -5295,7 +5374,7 @@ function connectionFromRow(row) {
|
|
|
5295
5374
|
projectId: row.project_id,
|
|
5296
5375
|
connectorId: row.connector_id,
|
|
5297
5376
|
authorizationId: row.authorization_id,
|
|
5298
|
-
owner:
|
|
5377
|
+
owner: connectorConnectionOwnerFromColumns(row.owner_type, row.owner_id),
|
|
5299
5378
|
slot: row.slot,
|
|
5300
5379
|
account: row.account,
|
|
5301
5380
|
status: row.status,
|
|
@@ -10228,6 +10307,386 @@ ALTER TABLE ai_model_call_usage
|
|
|
10228
10307
|
);
|
|
10229
10308
|
`;
|
|
10230
10309
|
|
|
10310
|
+
// src/migrations/089-maintenance-heartbeats.sql
|
|
10311
|
+
var _089_maintenance_heartbeats_default = `-- One row per project and owner, rewritten by that owner's maintenance loop as it runs. Another
|
|
10312
|
+
-- process reads it to tell a stopped loop from a quiet one.
|
|
10313
|
+
CREATE TABLE maintenance_heartbeats (
|
|
10314
|
+
project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
|
|
10315
|
+
owner TEXT NOT NULL CHECK (owner IN ('orchestrator', 'embedded')),
|
|
10316
|
+
beat_at TIMESTAMPTZ NOT NULL,
|
|
10317
|
+
every_ms INTEGER NOT NULL CHECK (every_ms > 0),
|
|
10318
|
+
passes JSONB NOT NULL CHECK (jsonb_typeof(passes) = 'array'),
|
|
10319
|
+
PRIMARY KEY (project_id, owner)
|
|
10320
|
+
);
|
|
10321
|
+
`;
|
|
10322
|
+
|
|
10323
|
+
// src/migrations/090-lake-retention-runs.sql
|
|
10324
|
+
var _090_lake_retention_runs_default = `-- One row per project, rewritten by each automatic lake retention run. The API reads it to tell a
|
|
10325
|
+
-- request nobody consumes from one a worker ran.
|
|
10326
|
+
CREATE TABLE lake_retention_runs (
|
|
10327
|
+
project_id TEXT PRIMARY KEY CHECK (length(trim(project_id)) > 0),
|
|
10328
|
+
started_at TIMESTAMPTZ NOT NULL,
|
|
10329
|
+
finished_at TIMESTAMPTZ,
|
|
10330
|
+
failed BOOLEAN NOT NULL
|
|
10331
|
+
);
|
|
10332
|
+
`;
|
|
10333
|
+
|
|
10334
|
+
// src/migrations/091-dataset-ingest-inbox.sql
|
|
10335
|
+
var _091_dataset_ingest_inbox_default = `-- Changes \`sixb.datasets.ingest\` stored and no flush has written to the lake yet. A flush tags the
|
|
10336
|
+
-- deliveries it takes with its id, writes them as one dataset version, then deletes them. One
|
|
10337
|
+
-- refused when written stays, marked, until the orchestrator purges it.
|
|
10338
|
+
CREATE TABLE dataset_ingest_deliveries (
|
|
10339
|
+
id BIGSERIAL,
|
|
10340
|
+
project_id TEXT NOT NULL,
|
|
10341
|
+
dataset_id TEXT NOT NULL,
|
|
10342
|
+
execution_id TEXT NOT NULL,
|
|
10343
|
+
correlation_id TEXT NOT NULL,
|
|
10344
|
+
received_at TIMESTAMPTZ NOT NULL,
|
|
10345
|
+
change_count INTEGER NOT NULL CHECK (change_count > 0),
|
|
10346
|
+
definition_hash TEXT NOT NULL,
|
|
10347
|
+
-- Text, not JSONB: nothing reads inside it, and JSONB refuses strings JSON allows.
|
|
10348
|
+
changes TEXT NOT NULL,
|
|
10349
|
+
flush_id TEXT,
|
|
10350
|
+
rejected_at TIMESTAMPTZ,
|
|
10351
|
+
rejection TEXT,
|
|
10352
|
+
-- Led by the dataset, so no plan for one dataset walks the deliveries of another.
|
|
10353
|
+
PRIMARY KEY (project_id, dataset_id, id)
|
|
10354
|
+
);
|
|
10355
|
+
CREATE INDEX dataset_ingest_deliveries_by_flush
|
|
10356
|
+
ON dataset_ingest_deliveries (project_id, dataset_id, flush_id, id);
|
|
10357
|
+
|
|
10358
|
+
-- One row per dataset: its open flush request, the lease of the flush that holds it, the batch
|
|
10359
|
+
-- that flush took, and when a batch last completed. Locking this row orders requests against batches.
|
|
10360
|
+
CREATE TABLE dataset_ingest_state (
|
|
10361
|
+
project_id TEXT NOT NULL,
|
|
10362
|
+
dataset_id TEXT NOT NULL,
|
|
10363
|
+
request BIGINT NOT NULL DEFAULT 0,
|
|
10364
|
+
requested_at TIMESTAMPTZ,
|
|
10365
|
+
lease_owner TEXT,
|
|
10366
|
+
lease_expires_at TIMESTAMPTZ,
|
|
10367
|
+
flush_id TEXT,
|
|
10368
|
+
flush_started_at TIMESTAMPTZ,
|
|
10369
|
+
failure_streak INTEGER NOT NULL DEFAULT 0,
|
|
10370
|
+
last_flushed_at TIMESTAMPTZ,
|
|
10371
|
+
PRIMARY KEY (project_id, dataset_id)
|
|
10372
|
+
);
|
|
10373
|
+
`;
|
|
10374
|
+
|
|
10375
|
+
// src/migrations/092-run-recovery.sql
|
|
10376
|
+
var _092_run_recovery_default = `-- A sync or pipeline run now survives its worker: each execution holds the run under a token that
|
|
10377
|
+
-- fences its writes, and projects its queue lease here so another delivery can tell a live holder
|
|
10378
|
+
-- from a dead one. A run with neither was started by an older version: nothing backfills it, and the
|
|
10379
|
+
-- recovery pass settles it once its old worker cannot still be alive.
|
|
10380
|
+
--
|
|
10381
|
+
-- Every column is added without touching a row. Constraints are \`NOT VALID\`: rows that exist cannot
|
|
10382
|
+
-- break them, and validating would scan the table under this lock.
|
|
10383
|
+
ALTER TABLE sync_runs
|
|
10384
|
+
ADD COLUMN attempt BIGINT NOT NULL DEFAULT 0,
|
|
10385
|
+
ADD COLUMN execution_token TEXT,
|
|
10386
|
+
ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ,
|
|
10387
|
+
ADD COLUMN commit_prepared_at TIMESTAMPTZ,
|
|
10388
|
+
ADD CONSTRAINT sync_runs_attempt_check CHECK (attempt >= 0) NOT VALID,
|
|
10389
|
+
ADD CONSTRAINT sync_runs_execution_lease_check
|
|
10390
|
+
CHECK ((execution_token IS NULL) = (execution_queue_lease_expires_at IS NULL)) NOT VALID;
|
|
10391
|
+
|
|
10392
|
+
ALTER TABLE pipeline_runs
|
|
10393
|
+
ADD COLUMN attempt BIGINT NOT NULL DEFAULT 0,
|
|
10394
|
+
ADD COLUMN execution_token TEXT,
|
|
10395
|
+
ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ,
|
|
10396
|
+
ADD CONSTRAINT pipeline_runs_attempt_check CHECK (attempt >= 0) NOT VALID,
|
|
10397
|
+
ADD CONSTRAINT pipeline_runs_execution_lease_check
|
|
10398
|
+
CHECK ((execution_token IS NULL) = (execution_queue_lease_expires_at IS NULL)) NOT VALID;
|
|
10399
|
+
`;
|
|
10400
|
+
|
|
10401
|
+
// src/migrations/093-webhook-and-projection-leases.sql
|
|
10402
|
+
var _093_webhook_and_projection_leases_default = `-- A webhook handler runs in the request, with no queue behind it, so the process running it keeps
|
|
10403
|
+
-- its own lease. A run with none was started by an older API, and is judged from when it started.
|
|
10404
|
+
-- A projection run holds a lease as other runs do, so a duplicate delivery leaves a live run alone.
|
|
10405
|
+
--
|
|
10406
|
+
-- These add nullable columns, which touches no row.
|
|
10407
|
+
ALTER TABLE webhook_runs
|
|
10408
|
+
ADD COLUMN execution_token TEXT,
|
|
10409
|
+
ADD COLUMN execution_lease_expires_at TIMESTAMPTZ;
|
|
10410
|
+
|
|
10411
|
+
ALTER TABLE projection_runs ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ;
|
|
10412
|
+
`;
|
|
10413
|
+
|
|
10414
|
+
// src/migrations/094-artifact-board-previews.sql
|
|
10415
|
+
var _094_artifact_board_previews_default = `-- A picture of a canvas's board in one mode and size, kept under its render key: a hash of what the board
|
|
10416
|
+
-- shows, which its version's outline names. Versions that keep a board as it was share its
|
|
10417
|
+
-- picture, so no artifact owns one and deleting an artifact leaves them. The first picture kept
|
|
10418
|
+
-- under a key stays.
|
|
10419
|
+
CREATE TABLE artifact_board_previews (
|
|
10420
|
+
project_id TEXT NOT NULL,
|
|
10421
|
+
render_key TEXT NOT NULL CHECK (length(render_key) > 0),
|
|
10422
|
+
theme TEXT NOT NULL CHECK (theme IN ('light', 'dark')),
|
|
10423
|
+
-- \`full\`, a PNG of the board at its own size; \`small\`, a WebP at most 480 pixels wide.
|
|
10424
|
+
size TEXT NOT NULL CHECK (size IN ('full', 'small')),
|
|
10425
|
+
file JSONB NOT NULL,
|
|
10426
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
10427
|
+
PRIMARY KEY (project_id, render_key, theme, size)
|
|
10428
|
+
);
|
|
10429
|
+
`;
|
|
10430
|
+
|
|
10431
|
+
// src/migrations/095-artifact-open-versions.sql
|
|
10432
|
+
var _095_artifact_open_versions_default = `-- An artifact's latest version is open until someone creates a version, which freezes it: a write
|
|
10433
|
+
-- replaces an open version's content, and the first write after a freeze opens the next version.
|
|
10434
|
+
-- \`revision\` counts every change to an artifact and only grows; a version's is the artifact's
|
|
10435
|
+
-- when it last changed. Every existing version was published to stay as it is, so each is frozen
|
|
10436
|
+
-- when it was published, and an address pinned to one keeps naming what it named.
|
|
10437
|
+
ALTER TABLE artifacts ADD COLUMN revision INTEGER;
|
|
10438
|
+
UPDATE artifacts SET revision = latest_version;
|
|
10439
|
+
ALTER TABLE artifacts
|
|
10440
|
+
ALTER COLUMN revision SET NOT NULL,
|
|
10441
|
+
ADD CONSTRAINT artifacts_revision_check CHECK (revision > 0);
|
|
10442
|
+
|
|
10443
|
+
-- \`created_at\` is when a version opened, \`updated_at\` when its content was last written, and
|
|
10444
|
+
-- \`frozen_at\` when someone created it, \`frozen_by_*\`: an existing version by whoever published it.
|
|
10445
|
+
ALTER TABLE artifact_versions
|
|
10446
|
+
ADD COLUMN revision INTEGER,
|
|
10447
|
+
ADD COLUMN updated_at TIMESTAMPTZ,
|
|
10448
|
+
ADD COLUMN frozen_at TIMESTAMPTZ,
|
|
10449
|
+
ADD COLUMN frozen_by_principal_type TEXT
|
|
10450
|
+
CHECK (frozen_by_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
10451
|
+
ADD COLUMN frozen_by_principal_id TEXT CHECK (length(frozen_by_principal_id) > 0);
|
|
10452
|
+
UPDATE artifact_versions
|
|
10453
|
+
SET revision = version, updated_at = created_at, frozen_at = created_at,
|
|
10454
|
+
frozen_by_principal_type = author_principal_type, frozen_by_principal_id = author_principal_id;
|
|
10455
|
+
ALTER TABLE artifact_versions
|
|
10456
|
+
ALTER COLUMN revision SET NOT NULL,
|
|
10457
|
+
ALTER COLUMN updated_at SET NOT NULL,
|
|
10458
|
+
ADD CONSTRAINT artifact_versions_revision_check CHECK (revision > 0),
|
|
10459
|
+
ADD CONSTRAINT artifact_versions_frozen_by_check CHECK (
|
|
10460
|
+
(frozen_at IS NULL) = (frozen_by_principal_type IS NULL)
|
|
10461
|
+
AND (frozen_by_principal_type IS NULL) = (frozen_by_principal_id IS NULL)
|
|
10462
|
+
);
|
|
10463
|
+
|
|
10464
|
+
-- At most one open version: the latest, which storage keeps so.
|
|
10465
|
+
CREATE UNIQUE INDEX idx_artifact_versions_open
|
|
10466
|
+
ON artifact_versions (project_id, artifact_id) WHERE frozen_at IS NULL;
|
|
10467
|
+
|
|
10468
|
+
-- A retry no longer finds what a turn wrote by its versions' origins, which the next write
|
|
10469
|
+
-- replaces: \`artifact_turn_writes\` keeps it.
|
|
10470
|
+
DROP INDEX idx_artifact_versions_origin_run;
|
|
10471
|
+
DROP INDEX idx_artifact_versions_origin_thread;
|
|
10472
|
+
|
|
10473
|
+
-- A frame link serves one revision of its version, and of each file it lends. The links issued
|
|
10474
|
+
-- before revisions name none, and expire within the hour: they go now, and the app asks again.
|
|
10475
|
+
DELETE FROM artifact_frame_grants;
|
|
10476
|
+
ALTER TABLE artifact_frame_grants
|
|
10477
|
+
ADD COLUMN revision INTEGER NOT NULL CHECK (revision > 0);
|
|
10478
|
+
|
|
10479
|
+
-- The Agent turn whose write shared an artifact in a thread: a retry of the turn takes the link
|
|
10480
|
+
-- back. Links made before turns were kept stay for as long as the artifact lasts.
|
|
10481
|
+
ALTER TABLE artifact_threads ADD COLUMN linked_by_run_id TEXT;
|
|
10482
|
+
CREATE INDEX idx_artifact_threads_linked_by_run
|
|
10483
|
+
ON artifact_threads (project_id, linked_by_run_id) WHERE linked_by_run_id IS NOT NULL;
|
|
10484
|
+
|
|
10485
|
+
-- What an artifact's latest version held before a change that can be undone: an Agent turn's
|
|
10486
|
+
-- first write to it, one per turn, or a restore. Kept by the revision whose content it holds; each
|
|
10487
|
+
-- artifact keeps its newest 20. \`refs\`, as in \`artifact_versions\`.
|
|
10488
|
+
CREATE TABLE artifact_restore_points (
|
|
10489
|
+
project_id TEXT NOT NULL,
|
|
10490
|
+
artifact_id TEXT NOT NULL,
|
|
10491
|
+
revision INTEGER NOT NULL CHECK (revision > 0),
|
|
10492
|
+
version INTEGER NOT NULL CHECK (version > 0),
|
|
10493
|
+
frozen BOOLEAN NOT NULL,
|
|
10494
|
+
title TEXT NOT NULL CHECK (length(trim(title)) > 0),
|
|
10495
|
+
entry TEXT NOT NULL CHECK (length(entry) > 0),
|
|
10496
|
+
files JSONB NOT NULL CHECK (jsonb_typeof(files) = 'array'),
|
|
10497
|
+
outline JSONB NOT NULL CHECK (jsonb_typeof(outline) = 'object'),
|
|
10498
|
+
refs JSONB NOT NULL CHECK (jsonb_typeof(refs) = 'array'),
|
|
10499
|
+
author_principal_type TEXT NOT NULL
|
|
10500
|
+
CHECK (author_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
10501
|
+
author_principal_id TEXT NOT NULL CHECK (length(author_principal_id) > 0),
|
|
10502
|
+
origin_thread_id TEXT,
|
|
10503
|
+
origin_run_id TEXT,
|
|
10504
|
+
updated_at TIMESTAMPTZ NOT NULL,
|
|
10505
|
+
reason TEXT NOT NULL CHECK (reason IN ('turn', 'restore')),
|
|
10506
|
+
turn_run_id TEXT,
|
|
10507
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
10508
|
+
PRIMARY KEY (project_id, artifact_id, revision),
|
|
10509
|
+
FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE,
|
|
10510
|
+
CHECK ((origin_thread_id IS NULL) = (origin_run_id IS NULL)),
|
|
10511
|
+
CHECK ((reason = 'turn') = (turn_run_id IS NOT NULL))
|
|
10512
|
+
);
|
|
10513
|
+
|
|
10514
|
+
CREATE UNIQUE INDEX idx_artifact_restore_points_turn
|
|
10515
|
+
ON artifact_restore_points (project_id, artifact_id, turn_run_id) WHERE turn_run_id IS NOT NULL;
|
|
10516
|
+
|
|
10517
|
+
-- The artifacts each Agent turn wrote, its sub-agents' writes among them, the revision its last
|
|
10518
|
+
-- write left and how many changes it made. Every change moves an artifact's revision by one, so a
|
|
10519
|
+
-- retry undoes an artifact only while those changes are all its revision moved by since the turn
|
|
10520
|
+
-- began: anyone's change in between keeps it. Each artifact keeps its newest 20 turns.
|
|
10521
|
+
CREATE TABLE artifact_turn_writes (
|
|
10522
|
+
project_id TEXT NOT NULL,
|
|
10523
|
+
artifact_id TEXT NOT NULL,
|
|
10524
|
+
turn_run_id TEXT NOT NULL CHECK (length(turn_run_id) > 0),
|
|
10525
|
+
created BOOLEAN NOT NULL,
|
|
10526
|
+
revision INTEGER NOT NULL CHECK (revision > 0),
|
|
10527
|
+
writes INTEGER NOT NULL CHECK (writes > 0),
|
|
10528
|
+
PRIMARY KEY (project_id, artifact_id, turn_run_id),
|
|
10529
|
+
FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE
|
|
10530
|
+
);
|
|
10531
|
+
|
|
10532
|
+
CREATE INDEX idx_artifact_turn_writes_turn ON artifact_turn_writes (project_id, turn_run_id);
|
|
10533
|
+
`;
|
|
10534
|
+
|
|
10535
|
+
// src/migrations/096-artifact-documents.sql
|
|
10536
|
+
var _096_artifact_documents_default = `-- A canvas's content as the Yjs document everyone editing it shares: its whole encoded state, in a
|
|
10537
|
+
-- blob, replaced by each write that changes the content, and the revision whose content it holds.
|
|
10538
|
+
-- It is not a journal of changes, only the document's own encoding, so a write merges into the
|
|
10539
|
+
-- same history wherever it is made. One whose revision is not the artifact's was left behind by a
|
|
10540
|
+
-- write that stored none: it is brought up to the content before it is used. Null until a write
|
|
10541
|
+
-- stores one, and for kinds that are not edited live.
|
|
10542
|
+
ALTER TABLE artifacts
|
|
10543
|
+
ADD COLUMN document JSONB CHECK (document IS NULL OR jsonb_typeof(document) = 'object'),
|
|
10544
|
+
ADD COLUMN document_revision INTEGER CHECK (document_revision > 0),
|
|
10545
|
+
ADD CONSTRAINT artifacts_document_pair_check
|
|
10546
|
+
CHECK ((document IS NULL) = (document_revision IS NULL));
|
|
10547
|
+
`;
|
|
10548
|
+
|
|
10549
|
+
// src/migrations/097-artifact-frame-files.sql
|
|
10550
|
+
var _097_artifact_frame_files_default = `-- A frame link names the files it serves, as a SHA-256 hex of their paths and digests, in place of
|
|
10551
|
+
-- the revision it was issued at: a canvas edited live is written every few seconds, and a link
|
|
10552
|
+
-- stays good while its boards' files are the same. Links last an hour, so those issued before are
|
|
10553
|
+
-- dropped.
|
|
10554
|
+
DELETE FROM artifact_frame_grants;
|
|
10555
|
+
ALTER TABLE artifact_frame_grants
|
|
10556
|
+
DROP COLUMN revision,
|
|
10557
|
+
ADD COLUMN files_digest TEXT NOT NULL CHECK (length(files_digest) = 64);
|
|
10558
|
+
`;
|
|
10559
|
+
|
|
10560
|
+
// src/migrations/098-artifact-comments.sql
|
|
10561
|
+
var _098_artifact_comments_default = `-- Comments on an artifact, and replies to them (\`parent_id\`), each about a board or an element of
|
|
10562
|
+
-- a canvas, or the artifact itself, at the revision its author saw (\`anchor\`). They go with the
|
|
10563
|
+
-- artifact, and a reply with what it replies to; a copy of the artifact keeps none.
|
|
10564
|
+
-- \`seq\` orders comments made in the same instant as they were made.
|
|
10565
|
+
CREATE TABLE artifact_comments (
|
|
10566
|
+
seq BIGINT GENERATED ALWAYS AS IDENTITY,
|
|
10567
|
+
project_id TEXT NOT NULL,
|
|
10568
|
+
artifact_id TEXT NOT NULL,
|
|
10569
|
+
id TEXT NOT NULL CHECK (length(id) > 0),
|
|
10570
|
+
parent_id TEXT,
|
|
10571
|
+
anchor JSONB NOT NULL CHECK (jsonb_typeof(anchor) = 'object'),
|
|
10572
|
+
body TEXT NOT NULL CHECK (length(body) > 0),
|
|
10573
|
+
author_principal_type TEXT NOT NULL
|
|
10574
|
+
CHECK (author_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
10575
|
+
author_principal_id TEXT NOT NULL CHECK (length(author_principal_id) > 0),
|
|
10576
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
10577
|
+
resolved_at TIMESTAMPTZ,
|
|
10578
|
+
resolved_by_principal_type TEXT
|
|
10579
|
+
CHECK (resolved_by_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
10580
|
+
resolved_by_principal_id TEXT CHECK (length(resolved_by_principal_id) > 0),
|
|
10581
|
+
-- The Agent's run that wrote it for its author, and that run's conversation.
|
|
10582
|
+
origin_thread_id TEXT,
|
|
10583
|
+
origin_run_id TEXT,
|
|
10584
|
+
PRIMARY KEY (project_id, artifact_id, id),
|
|
10585
|
+
FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE,
|
|
10586
|
+
FOREIGN KEY (project_id, artifact_id, parent_id)
|
|
10587
|
+
REFERENCES artifact_comments (project_id, artifact_id, id) ON DELETE CASCADE,
|
|
10588
|
+
CHECK (
|
|
10589
|
+
(resolved_at IS NULL) = (resolved_by_principal_type IS NULL)
|
|
10590
|
+
AND (resolved_by_principal_type IS NULL) = (resolved_by_principal_id IS NULL)
|
|
10591
|
+
),
|
|
10592
|
+
CHECK ((origin_thread_id IS NULL) = (origin_run_id IS NULL))
|
|
10593
|
+
);
|
|
10594
|
+
|
|
10595
|
+
CREATE INDEX idx_artifact_comments_created
|
|
10596
|
+
ON artifact_comments (project_id, artifact_id, created_at, seq);
|
|
10597
|
+
`;
|
|
10598
|
+
|
|
10599
|
+
// src/migrations/099-object-artifact-refs.sql
|
|
10600
|
+
var _099_object_artifact_refs_default = `-- The artifacts objects hold: one row per artifact an object property holds, wherever it sits in
|
|
10601
|
+
-- the property's value. Every object write keeps them current in its own transaction, so whoever
|
|
10602
|
+
-- reaches an artifact through the objects that hold it finds them here. Holders list by code
|
|
10603
|
+
-- point, which the "C" collation keeps in the key's order.
|
|
10604
|
+
CREATE TABLE object_artifact_refs (
|
|
10605
|
+
project_id TEXT NOT NULL,
|
|
10606
|
+
artifact_id TEXT NOT NULL,
|
|
10607
|
+
object_type_id TEXT COLLATE "C" NOT NULL,
|
|
10608
|
+
primary_id TEXT COLLATE "C" NOT NULL,
|
|
10609
|
+
property_id TEXT COLLATE "C" NOT NULL,
|
|
10610
|
+
PRIMARY KEY (project_id, artifact_id, object_type_id, primary_id, property_id)
|
|
10611
|
+
);
|
|
10612
|
+
`;
|
|
10613
|
+
|
|
10614
|
+
// src/migrations/100-shared-link-guests.sql
|
|
10615
|
+
var _100_shared_link_guests_default = `-- A shared link's visitor has no account: they give a name once, which their session keeps.
|
|
10616
|
+
ALTER TABLE share_sessions
|
|
10617
|
+
ADD COLUMN guest_name TEXT CHECK (guest_name IS NULL OR length(guest_name) > 0);
|
|
10618
|
+
|
|
10619
|
+
-- They comment as a guest: the author's id is their session, and the comment keeps the name they
|
|
10620
|
+
-- gave, whatever becomes of the session.
|
|
10621
|
+
ALTER TABLE artifact_comments
|
|
10622
|
+
DROP CONSTRAINT artifact_comments_author_principal_type_check;
|
|
10623
|
+
|
|
10624
|
+
ALTER TABLE artifact_comments
|
|
10625
|
+
ADD COLUMN author_name TEXT,
|
|
10626
|
+
ADD CONSTRAINT artifact_comments_author_principal_type_check
|
|
10627
|
+
CHECK (author_principal_type IN ('user', 'serviceAccount', 'system', 'guest')),
|
|
10628
|
+
ADD CONSTRAINT artifact_comments_author_name_check CHECK (
|
|
10629
|
+
(author_principal_type = 'guest') = (author_name IS NOT NULL)
|
|
10630
|
+
AND (author_name IS NULL OR length(author_name) > 0)
|
|
10631
|
+
);
|
|
10632
|
+
`;
|
|
10633
|
+
|
|
10634
|
+
// src/migrations/101-connector-connection-owners.sql
|
|
10635
|
+
var _101_connector_connection_owners_default = `-- A connection belongs to the project or to one person. The project's owner_id is empty, so the
|
|
10636
|
+
-- unique key over (owner_type, owner_id, slot) never meets a NULL. Existing rows become the
|
|
10637
|
+
-- project's; the defaults are dropped afterwards so every write has to name its owner.
|
|
10638
|
+
ALTER TABLE connector_connections
|
|
10639
|
+
ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
|
|
10640
|
+
ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
|
|
10641
|
+
|
|
10642
|
+
ALTER TABLE connector_connections
|
|
10643
|
+
ALTER COLUMN owner_type DROP DEFAULT,
|
|
10644
|
+
ALTER COLUMN owner_id DROP DEFAULT,
|
|
10645
|
+
ADD CONSTRAINT connector_connections_owner_check CHECK (
|
|
10646
|
+
(owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
|
|
10647
|
+
),
|
|
10648
|
+
DROP CONSTRAINT connector_connections_project_id_connector_id_slot_key,
|
|
10649
|
+
ADD CONSTRAINT connector_connections_owner_slot_key
|
|
10650
|
+
UNIQUE (project_id, connector_id, owner_type, owner_id, slot);
|
|
10651
|
+
|
|
10652
|
+
ALTER TABLE connector_connection_runs
|
|
10653
|
+
ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
|
|
10654
|
+
ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
|
|
10655
|
+
|
|
10656
|
+
ALTER TABLE connector_connection_runs
|
|
10657
|
+
ALTER COLUMN owner_type DROP DEFAULT,
|
|
10658
|
+
ALTER COLUMN owner_id DROP DEFAULT,
|
|
10659
|
+
ADD CONSTRAINT connector_connection_runs_owner_check CHECK (
|
|
10660
|
+
(owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
|
|
10661
|
+
);
|
|
10662
|
+
|
|
10663
|
+
ALTER TABLE connector_authorization_attempts
|
|
10664
|
+
ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
|
|
10665
|
+
ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
|
|
10666
|
+
|
|
10667
|
+
ALTER TABLE connector_authorization_attempts
|
|
10668
|
+
ALTER COLUMN owner_type DROP DEFAULT,
|
|
10669
|
+
ALTER COLUMN owner_id DROP DEFAULT,
|
|
10670
|
+
ADD CONSTRAINT connector_authorization_attempts_owner_check CHECK (
|
|
10671
|
+
(owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
|
|
10672
|
+
);
|
|
10673
|
+
`;
|
|
10674
|
+
|
|
10675
|
+
// src/migrations/102-user-suspension-source.sql
|
|
10676
|
+
var _102_user_suspension_source_default = `-- Who suspended a user: an administrator or server code ('manual'), or the identity provider's
|
|
10677
|
+
-- directory ('directory'), which may lift only its own suspensions. Only a suspended user has one,
|
|
10678
|
+
-- and every suspension made before this column existed was manual.
|
|
10679
|
+
ALTER TABLE auth_users ADD COLUMN suspension_source TEXT;
|
|
10680
|
+
|
|
10681
|
+
UPDATE auth_users SET suspension_source = 'manual' WHERE status = 'suspended';
|
|
10682
|
+
|
|
10683
|
+
ALTER TABLE auth_users
|
|
10684
|
+
ADD CONSTRAINT auth_users_suspension_source_check CHECK (
|
|
10685
|
+
(status = 'suspended' AND suspension_source IN ('manual', 'directory'))
|
|
10686
|
+
OR (status = 'active' AND suspension_source IS NULL)
|
|
10687
|
+
);
|
|
10688
|
+
`;
|
|
10689
|
+
|
|
10231
10690
|
// src/migrations.ts
|
|
10232
10691
|
var POSTGRES_STORAGE_ADAPTER_ID = "SixbPostgresStorage";
|
|
10233
10692
|
function pgSql(id, sqlText) {
|
|
@@ -10482,7 +10941,21 @@ var postgresStorageMigrations = defineMigrations({
|
|
|
10482
10941
|
pgSql("085-agent-computer-sessions", _085_agent_computer_sessions_default),
|
|
10483
10942
|
pgSql("086-partition-objects", _086_partition_objects_default),
|
|
10484
10943
|
pgSql("087-agent-computer-background-commands", _087_agent_computer_background_commands_default),
|
|
10485
|
-
pgSql("088-web-search-model-kind", _088_web_search_model_kind_default)
|
|
10944
|
+
pgSql("088-web-search-model-kind", _088_web_search_model_kind_default),
|
|
10945
|
+
pgSql("089-maintenance-heartbeats", _089_maintenance_heartbeats_default),
|
|
10946
|
+
pgSql("090-lake-retention-runs", _090_lake_retention_runs_default),
|
|
10947
|
+
pgSql("091-dataset-ingest-inbox", _091_dataset_ingest_inbox_default),
|
|
10948
|
+
pgSql("092-run-recovery", _092_run_recovery_default),
|
|
10949
|
+
pgSql("093-webhook-and-projection-leases", _093_webhook_and_projection_leases_default),
|
|
10950
|
+
pgSql("094-artifact-board-previews", _094_artifact_board_previews_default),
|
|
10951
|
+
pgSql("095-artifact-open-versions", _095_artifact_open_versions_default),
|
|
10952
|
+
pgSql("096-artifact-documents", _096_artifact_documents_default),
|
|
10953
|
+
pgSql("097-artifact-frame-files", _097_artifact_frame_files_default),
|
|
10954
|
+
pgSql("098-artifact-comments", _098_artifact_comments_default),
|
|
10955
|
+
pgSql("099-object-artifact-refs", _099_object_artifact_refs_default),
|
|
10956
|
+
pgSql("100-shared-link-guests", _100_shared_link_guests_default),
|
|
10957
|
+
pgSql("101-connector-connection-owners", _101_connector_connection_owners_default),
|
|
10958
|
+
pgSql("102-user-suspension-source", _102_user_suspension_source_default)
|
|
10486
10959
|
]
|
|
10487
10960
|
});
|
|
10488
10961
|
|
|
@@ -14095,7 +14568,8 @@ var PG_OBJECT_QUERY_CAPABILITIES = {
|
|
|
14095
14568
|
decimal: { equality: true, ordering: true },
|
|
14096
14569
|
date: { equality: true, ordering: true },
|
|
14097
14570
|
timestamp: { equality: true, ordering: true },
|
|
14098
|
-
userRef: { equality: true }
|
|
14571
|
+
userRef: { equality: true },
|
|
14572
|
+
artifactRef: { equality: true }
|
|
14099
14573
|
},
|
|
14100
14574
|
limits: {
|
|
14101
14575
|
totalCount: true,
|
|
@@ -14208,6 +14682,21 @@ class PgObjectStorage {
|
|
|
14208
14682
|
...hasMore && last ? { nextPrimaryId: last.primaryId } : {}
|
|
14209
14683
|
};
|
|
14210
14684
|
}
|
|
14685
|
+
async listArtifactHolders(params) {
|
|
14686
|
+
assertArtifactHolderPageLimit(params.limit);
|
|
14687
|
+
const { after } = params;
|
|
14688
|
+
const rows = await this.sql`
|
|
14689
|
+
SELECT object_type_id, primary_id, property_id FROM object_artifact_refs
|
|
14690
|
+
WHERE project_id = ${params.projectId} AND artifact_id = ${params.artifactId}
|
|
14691
|
+
${after ? this.sql`AND (object_type_id, primary_id, property_id)
|
|
14692
|
+
> (${after.objectTypeId}, ${after.primaryId}, ${after.propertyId})` : this.sql``}
|
|
14693
|
+
ORDER BY object_type_id, primary_id, property_id
|
|
14694
|
+
LIMIT ${params.limit + 1}
|
|
14695
|
+
`;
|
|
14696
|
+
const holders = rows.slice(0, params.limit).map(artifactHolderFromRow);
|
|
14697
|
+
const last = holders.at(-1);
|
|
14698
|
+
return { holders, ...rows.length > params.limit && last ? { next: last } : {} };
|
|
14699
|
+
}
|
|
14211
14700
|
async list(params) {
|
|
14212
14701
|
return this.reader.list(params);
|
|
14213
14702
|
}
|
|
@@ -14233,6 +14722,18 @@ function assertReconciliationPageLimit(limit) {
|
|
|
14233
14722
|
throw new Error("Object reconciliation page limit must be a positive safe integer.");
|
|
14234
14723
|
}
|
|
14235
14724
|
}
|
|
14725
|
+
function artifactHolderFromRow(row) {
|
|
14726
|
+
return {
|
|
14727
|
+
objectTypeId: row.object_type_id,
|
|
14728
|
+
primaryId: row.primary_id,
|
|
14729
|
+
propertyId: row.property_id
|
|
14730
|
+
};
|
|
14731
|
+
}
|
|
14732
|
+
function assertArtifactHolderPageLimit(limit) {
|
|
14733
|
+
if (!Number.isSafeInteger(limit) || limit <= 0) {
|
|
14734
|
+
throw new Error("Artifact holder page limit must be a positive safe integer.");
|
|
14735
|
+
}
|
|
14736
|
+
}
|
|
14236
14737
|
// src/objects/prepare-queries.ts
|
|
14237
14738
|
import { createHash as createHash7 } from "crypto";
|
|
14238
14739
|
async function preparePgObjectQueries(sql, schema, plan) {
|
|
@@ -16358,6 +16859,7 @@ class PgMaterializationSessionState extends ProviderMaterializationSessionState
|
|
|
16358
16859
|
changedLinks = 0;
|
|
16359
16860
|
changedUniqueValues = 0;
|
|
16360
16861
|
uniqueWork = { releases: false, claims: false };
|
|
16862
|
+
artifactHolderWork = { deletes: false, inserts: false };
|
|
16361
16863
|
workAnalyzed = false;
|
|
16362
16864
|
workTable;
|
|
16363
16865
|
workId;
|
|
@@ -16461,6 +16963,10 @@ class PgMaterializationSessions {
|
|
|
16461
16963
|
session.uniqueWork.releases = true;
|
|
16462
16964
|
if (record.item.kind === "unique-value-claim")
|
|
16463
16965
|
session.uniqueWork.claims = true;
|
|
16966
|
+
if (record.item.kind === "artifact-holder-delete")
|
|
16967
|
+
session.artifactHolderWork.deletes = true;
|
|
16968
|
+
if (record.item.kind === "artifact-holder-insert")
|
|
16969
|
+
session.artifactHolderWork.inserts = true;
|
|
16464
16970
|
}
|
|
16465
16971
|
}
|
|
16466
16972
|
async analyzeSealedWork(session) {
|
|
@@ -16755,13 +17261,18 @@ class PgMaterializationWriter {
|
|
|
16755
17261
|
await this.applyLinkOverrides(target);
|
|
16756
17262
|
await this.applyPoints(target);
|
|
16757
17263
|
const uniqueWork = target.uniqueWork ?? await this.stagedUniqueWork(target);
|
|
17264
|
+
const holderWork = target.artifactHolderWork ?? await this.stagedArtifactHolderWork(target);
|
|
16758
17265
|
if (uniqueWork.releases)
|
|
16759
17266
|
await this.releaseUniqueValues(target);
|
|
17267
|
+
if (holderWork.deletes)
|
|
17268
|
+
await this.deleteArtifactHolders(target);
|
|
16760
17269
|
let linkWrites = await this.deleteLinks(target);
|
|
16761
17270
|
let objectWrites = await this.deleteObjects(target);
|
|
16762
17271
|
objectWrites += await this.upsertObjects(target);
|
|
16763
17272
|
linkWrites += await this.upsertLinks(target);
|
|
16764
17273
|
const uniqueValueWrites = uniqueWork.claims ? await this.claimUniqueValues(target) : 0;
|
|
17274
|
+
if (holderWork.inserts)
|
|
17275
|
+
await this.insertArtifactHolders(target);
|
|
16765
17276
|
const eventCount = await this.writeOutbox(target);
|
|
16766
17277
|
await this.writeTouches(target);
|
|
16767
17278
|
return { objectWrites, linkWrites, uniqueValueWrites, eventCount };
|
|
@@ -16792,6 +17303,21 @@ class PgMaterializationWriter {
|
|
|
16792
17303
|
`;
|
|
16793
17304
|
return { releases: row?.releases ?? false, claims: row?.claims ?? false };
|
|
16794
17305
|
}
|
|
17306
|
+
async stagedArtifactHolderWork(target) {
|
|
17307
|
+
const probe = (kind) => this.sql`
|
|
17308
|
+
EXISTS (
|
|
17309
|
+
SELECT 1 FROM ${this.sql(target.workTable)}
|
|
17310
|
+
WHERE work_id = ${target.workId} AND lane = 'apply'
|
|
17311
|
+
AND major_order = ${materializationApplyPhase2(kind)}
|
|
17312
|
+
AND minor_order = ${materializationPlanKindRank2(kind)}
|
|
17313
|
+
)
|
|
17314
|
+
`;
|
|
17315
|
+
const [row] = await this.sql`
|
|
17316
|
+
SELECT ${probe("artifact-holder-delete")} AS deletes,
|
|
17317
|
+
${probe("artifact-holder-insert")} AS inserts
|
|
17318
|
+
`;
|
|
17319
|
+
return { deletes: row?.deletes ?? false, inserts: row?.inserts ?? false };
|
|
17320
|
+
}
|
|
16795
17321
|
staged(target, kind) {
|
|
16796
17322
|
return this.sql`
|
|
16797
17323
|
SELECT payload->'item'->'value' AS value
|
|
@@ -17054,6 +17580,34 @@ class PgMaterializationWriter {
|
|
|
17054
17580
|
async claimUniqueValues(target) {
|
|
17055
17581
|
return claimPgUniqueValues(this.sql, target.projectId, this.stagedUniqueValues(target, "unique-value-claim"), { requirePrepared: true });
|
|
17056
17582
|
}
|
|
17583
|
+
stagedArtifactHolders(target, kind) {
|
|
17584
|
+
return this.sql`
|
|
17585
|
+
SELECT value->>'artifactId' AS artifact_id, value->>'objectTypeId' AS object_type_id,
|
|
17586
|
+
value->>'primaryId' AS primary_id, value->>'propertyId' AS property_id
|
|
17587
|
+
FROM (${this.staged(target, kind)}) AS staged
|
|
17588
|
+
`;
|
|
17589
|
+
}
|
|
17590
|
+
async deleteArtifactHolders(target) {
|
|
17591
|
+
await this.sql`
|
|
17592
|
+
WITH candidates AS (${this.stagedArtifactHolders(target, "artifact-holder-delete")})
|
|
17593
|
+
DELETE FROM object_artifact_refs AS stored USING candidates
|
|
17594
|
+
WHERE stored.project_id = ${target.projectId}
|
|
17595
|
+
AND stored.artifact_id = candidates.artifact_id
|
|
17596
|
+
AND stored.object_type_id = candidates.object_type_id
|
|
17597
|
+
AND stored.primary_id = candidates.primary_id
|
|
17598
|
+
AND stored.property_id = candidates.property_id
|
|
17599
|
+
`;
|
|
17600
|
+
}
|
|
17601
|
+
async insertArtifactHolders(target) {
|
|
17602
|
+
await this.sql`
|
|
17603
|
+
INSERT INTO object_artifact_refs (
|
|
17604
|
+
project_id, artifact_id, object_type_id, primary_id, property_id
|
|
17605
|
+
)
|
|
17606
|
+
SELECT ${target.projectId}, artifact_id, object_type_id, primary_id, property_id
|
|
17607
|
+
FROM (${this.stagedArtifactHolders(target, "artifact-holder-insert")}) AS candidates
|
|
17608
|
+
ON CONFLICT DO NOTHING
|
|
17609
|
+
`;
|
|
17610
|
+
}
|
|
17057
17611
|
async deleteLinks(target) {
|
|
17058
17612
|
const candidates = this.staged(target, "link-delete");
|
|
17059
17613
|
return this.write(this.sql`
|
|
@@ -17250,7 +17804,8 @@ class PgMaterializationWriter {
|
|
|
17250
17804
|
END AS ref
|
|
17251
17805
|
) AS entity
|
|
17252
17806
|
WHERE work.work_id = ${target.workId} AND work.lane = 'apply'
|
|
17253
|
-
-- Unique values are no plan input: a stale claim fails when it applies.
|
|
17807
|
+
-- Unique values are no plan input: a stale claim fails when it applies. Nor are the
|
|
17808
|
+
-- artifacts objects hold, which follow the objects.
|
|
17254
17809
|
AND entity.kind IS NOT NULL
|
|
17255
17810
|
`;
|
|
17256
17811
|
}
|
|
@@ -17448,7 +18003,7 @@ class PgOntologyMaterializationStorage {
|
|
|
17448
18003
|
projectId: commit.projectId,
|
|
17449
18004
|
commitId: commit.id,
|
|
17450
18005
|
committedAt: commit.committedAt,
|
|
17451
|
-
...session.plan ? {} : { uniqueWork: session.uniqueWork }
|
|
18006
|
+
...session.plan ? {} : { uniqueWork: session.uniqueWork, artifactHolderWork: session.artifactHolderWork }
|
|
17452
18007
|
});
|
|
17453
18008
|
if (applied.objectWrites >= STATISTICS_MINIMUM_CHANGES) {
|
|
17454
18009
|
session.changedObjects = await this.writer.stagedObjectWritesByType({
|
|
@@ -21417,30 +21972,62 @@ function invalidState(reservation, operation) {
|
|
|
21417
21972
|
}
|
|
21418
21973
|
|
|
21419
21974
|
// src/pg-artifact-storage.ts
|
|
21975
|
+
import { ARTIFACT_COMMENT_LIMITS } from "@sixb/core/artifacts";
|
|
21420
21976
|
import {
|
|
21977
|
+
ARTIFACT_RESTORE_POINTS,
|
|
21421
21978
|
artifactStateBytes,
|
|
21422
21979
|
artifactStorageErrors,
|
|
21423
21980
|
checkArtifactStateWrite,
|
|
21981
|
+
othersCommented,
|
|
21982
|
+
parseArtifactBoardPreviewRow,
|
|
21983
|
+
parseArtifactCommentRow,
|
|
21424
21984
|
parseArtifactFrameGrantRow,
|
|
21985
|
+
parseArtifactRestorePointRow,
|
|
21425
21986
|
parseArtifactRow,
|
|
21426
21987
|
parseArtifactStateRow,
|
|
21427
|
-
|
|
21988
|
+
parseArtifactTurnWriteRow,
|
|
21989
|
+
parseArtifactVersionRow,
|
|
21990
|
+
turnUndoable
|
|
21428
21991
|
} from "@sixb/core/internal/storage";
|
|
21429
21992
|
var ARTIFACT_COLUMNS = `
|
|
21430
21993
|
project_id AS "projectId", id, kind, title,
|
|
21431
21994
|
owner_principal_type AS "ownerPrincipalType", owner_principal_id AS "ownerPrincipalId",
|
|
21432
|
-
latest_version AS "latestVersion", created_at AS "createdAt",
|
|
21995
|
+
latest_version AS "latestVersion", revision, created_at AS "createdAt",
|
|
21996
|
+
updated_at AS "updatedAt", document, document_revision AS "documentRevision"
|
|
21997
|
+
`;
|
|
21998
|
+
var CONTENT_COLUMNS = `
|
|
21999
|
+
title, entry, files, outline, refs, author_principal_type AS "authorPrincipalType",
|
|
22000
|
+
author_principal_id AS "authorPrincipalId", origin_thread_id AS "originThreadId",
|
|
22001
|
+
origin_run_id AS "originRunId", updated_at AS "updatedAt"
|
|
21433
22002
|
`;
|
|
21434
22003
|
var VERSION_COLUMNS = `
|
|
21435
|
-
project_id AS "projectId", artifact_id AS "artifactId", version,
|
|
21436
|
-
|
|
21437
|
-
|
|
22004
|
+
project_id AS "projectId", artifact_id AS "artifactId", version, revision, ${CONTENT_COLUMNS},
|
|
22005
|
+
created_at AS "createdAt", frozen_at AS "frozenAt",
|
|
22006
|
+
frozen_by_principal_type AS "frozenByPrincipalType",
|
|
22007
|
+
frozen_by_principal_id AS "frozenByPrincipalId"
|
|
22008
|
+
`;
|
|
22009
|
+
var RESTORE_POINT_COLUMNS = `
|
|
22010
|
+
project_id AS "projectId", artifact_id AS "artifactId", revision, version, frozen,
|
|
22011
|
+
${CONTENT_COLUMNS}, reason, turn_run_id AS "turnRunId", created_at AS "createdAt"
|
|
21438
22012
|
`;
|
|
22013
|
+
var TURN_WRITE_COLUMNS = `artifact_id AS "artifactId", created, revision, writes`;
|
|
21439
22014
|
var FRAME_GRANT_COLUMNS = `
|
|
21440
22015
|
project_id AS "projectId", token_hash AS "tokenHash", artifact_id AS "artifactId", version,
|
|
21441
|
-
targets, created_at AS "createdAt", expires_at AS "expiresAt"
|
|
22016
|
+
files_digest AS "filesDigest", targets, created_at AS "createdAt", expires_at AS "expiresAt"
|
|
22017
|
+
`;
|
|
22018
|
+
var COMMENT_COLUMNS = `
|
|
22019
|
+
project_id AS "projectId", artifact_id AS "artifactId", id, parent_id AS "parentId", anchor,
|
|
22020
|
+
body, author_principal_type AS "authorPrincipalType", author_principal_id AS "authorPrincipalId",
|
|
22021
|
+
author_name AS "authorName", created_at AS "createdAt", resolved_at AS "resolvedAt",
|
|
22022
|
+
resolved_by_principal_type AS "resolvedByPrincipalType",
|
|
22023
|
+
resolved_by_principal_id AS "resolvedByPrincipalId",
|
|
22024
|
+
origin_thread_id AS "originThreadId", origin_run_id AS "originRunId"
|
|
21442
22025
|
`;
|
|
21443
22026
|
var STATE_COLUMNS = `key, value, size_bytes AS "sizeBytes", updated_at AS "updatedAt"`;
|
|
22027
|
+
var BOARD_PREVIEW_COLUMNS = `
|
|
22028
|
+
project_id AS "projectId", render_key AS "renderKey", theme, size, file,
|
|
22029
|
+
created_at AS "createdAt"
|
|
22030
|
+
`;
|
|
21444
22031
|
|
|
21445
22032
|
class PgArtifactStorage {
|
|
21446
22033
|
sql;
|
|
@@ -21449,61 +22036,223 @@ class PgArtifactStorage {
|
|
|
21449
22036
|
}
|
|
21450
22037
|
async create(input) {
|
|
21451
22038
|
return runPgTransaction(this.sql, async (tx) => {
|
|
22039
|
+
const content = input.version;
|
|
21452
22040
|
const [artifact] = await tx`
|
|
21453
22041
|
INSERT INTO artifacts (
|
|
21454
22042
|
project_id, id, kind, title, owner_principal_type, owner_principal_id,
|
|
21455
|
-
latest_version, next_version, created_at, updated_at
|
|
22043
|
+
latest_version, next_version, revision, created_at, updated_at, document,
|
|
22044
|
+
document_revision
|
|
21456
22045
|
) VALUES (
|
|
21457
|
-
${input.projectId}, ${input.id}, ${input.kind}, ${
|
|
21458
|
-
${input.owner.type}, ${input.owner.id}, 1, 2,
|
|
21459
|
-
${
|
|
22046
|
+
${input.projectId}, ${input.id}, ${input.kind}, ${content.title},
|
|
22047
|
+
${input.owner.type}, ${input.owner.id}, 1, 2, 1,
|
|
22048
|
+
${content.updatedAt}, ${content.updatedAt}, ${documentJson(input.document)}::text::jsonb,
|
|
22049
|
+
${input.document ? 1 : null}
|
|
21460
22050
|
)
|
|
21461
22051
|
ON CONFLICT (project_id, id) DO NOTHING
|
|
21462
22052
|
RETURNING ${tx.unsafe(ARTIFACT_COLUMNS)}
|
|
21463
22053
|
`;
|
|
21464
22054
|
if (!artifact)
|
|
21465
22055
|
throw artifactStorageErrors.exists(input.id);
|
|
21466
|
-
|
|
21467
|
-
|
|
21468
|
-
|
|
21469
|
-
|
|
22056
|
+
const version = await insertVersion(tx, {
|
|
22057
|
+
projectId: input.projectId,
|
|
22058
|
+
artifactId: input.id,
|
|
22059
|
+
version: 1,
|
|
22060
|
+
revision: 1,
|
|
22061
|
+
content
|
|
22062
|
+
});
|
|
22063
|
+
await shareIn(tx, input.projectId, input.id, content, input.turnRunId);
|
|
22064
|
+
if (input.turnRunId) {
|
|
22065
|
+
await recordTurnWrite(tx, {
|
|
22066
|
+
projectId: input.projectId,
|
|
22067
|
+
artifactId: input.id,
|
|
22068
|
+
turnRunId: input.turnRunId,
|
|
22069
|
+
created: true,
|
|
22070
|
+
revision: 1
|
|
22071
|
+
});
|
|
22072
|
+
}
|
|
22073
|
+
return { artifact: parseArtifactRow(artifact), version };
|
|
21470
22074
|
});
|
|
21471
22075
|
}
|
|
21472
22076
|
async publish(input) {
|
|
21473
22077
|
return runPgTransaction(this.sql, async (tx) => {
|
|
21474
|
-
const
|
|
21475
|
-
SELECT kind, next_version, latest_version FROM artifacts
|
|
21476
|
-
WHERE project_id = ${input.projectId} AND id = ${input.artifactId}
|
|
21477
|
-
FOR UPDATE
|
|
21478
|
-
`;
|
|
22078
|
+
const current = await lockArtifact(tx, input.projectId, input.artifactId);
|
|
21479
22079
|
if (!current)
|
|
21480
22080
|
throw artifactStorageErrors.notFound(input.artifactId);
|
|
21481
|
-
const
|
|
21482
|
-
if (kind !== input.kind) {
|
|
21483
|
-
throw artifactStorageErrors.kindMismatch(input.artifactId, kind, input.kind);
|
|
22081
|
+
const { artifact } = current;
|
|
22082
|
+
if (artifact.kind !== input.kind) {
|
|
22083
|
+
throw artifactStorageErrors.kindMismatch(input.artifactId, artifact.kind, input.kind);
|
|
21484
22084
|
}
|
|
21485
|
-
const
|
|
21486
|
-
|
|
21487
|
-
|
|
21488
|
-
|
|
21489
|
-
|
|
21490
|
-
|
|
21491
|
-
|
|
21492
|
-
|
|
21493
|
-
throw artifactStorageErrors.moved(input.artifactId, input.baseVersion, {
|
|
21494
|
-
version: latest.version,
|
|
21495
|
-
author: latest.author
|
|
22085
|
+
const latest = await latestVersion(tx, artifact);
|
|
22086
|
+
assertBase(artifact, latest, input.baseRevision);
|
|
22087
|
+
const { turnRunId } = input;
|
|
22088
|
+
if (turnRunId && !await turnWrote(tx, artifact, turnRunId)) {
|
|
22089
|
+
await keepRestorePoint(tx, latest, {
|
|
22090
|
+
reason: "turn",
|
|
22091
|
+
turnRunId,
|
|
22092
|
+
createdAt: input.version.updatedAt
|
|
21496
22093
|
});
|
|
21497
22094
|
}
|
|
21498
|
-
const
|
|
21499
|
-
|
|
22095
|
+
const written = await writeLatest(tx, current, latest, input.version, input.document);
|
|
22096
|
+
await shareIn(tx, input.projectId, input.artifactId, input.version, turnRunId);
|
|
22097
|
+
if (turnRunId) {
|
|
22098
|
+
await recordTurnWrite(tx, {
|
|
22099
|
+
projectId: input.projectId,
|
|
22100
|
+
artifactId: input.artifactId,
|
|
22101
|
+
turnRunId,
|
|
22102
|
+
created: false,
|
|
22103
|
+
revision: written.artifact.revision
|
|
22104
|
+
});
|
|
22105
|
+
}
|
|
22106
|
+
return written;
|
|
22107
|
+
});
|
|
22108
|
+
}
|
|
22109
|
+
async freeze(input) {
|
|
22110
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
22111
|
+
const current = await lockArtifact(tx, input.projectId, input.artifactId);
|
|
22112
|
+
if (!current)
|
|
22113
|
+
throw artifactStorageErrors.notFound(input.artifactId);
|
|
22114
|
+
const { artifact } = current;
|
|
22115
|
+
const latest = await latestVersion(tx, artifact);
|
|
22116
|
+
if (latest.frozenAt)
|
|
22117
|
+
return { artifact, version: latest };
|
|
22118
|
+
assertBase(artifact, latest, input.baseRevision);
|
|
22119
|
+
const revision = artifact.revision + 1;
|
|
22120
|
+
const [version] = await tx`
|
|
22121
|
+
UPDATE artifact_versions
|
|
22122
|
+
SET revision = ${revision}, frozen_at = ${input.frozenAt},
|
|
22123
|
+
frozen_by_principal_type = ${input.frozenBy.type},
|
|
22124
|
+
frozen_by_principal_id = ${input.frozenBy.id}
|
|
22125
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22126
|
+
AND version = ${latest.version}
|
|
22127
|
+
RETURNING ${tx.unsafe(VERSION_COLUMNS)}
|
|
22128
|
+
`;
|
|
22129
|
+
const [updated] = await tx`
|
|
21500
22130
|
UPDATE artifacts
|
|
21501
|
-
SET
|
|
21502
|
-
|
|
22131
|
+
SET revision = ${revision}, updated_at = ${input.frozenAt},
|
|
22132
|
+
-- A freeze keeps the content, so a document that held it still does.
|
|
22133
|
+
document_revision = CASE WHEN document_revision = ${artifact.revision}::int
|
|
22134
|
+
THEN ${revision}::int ELSE document_revision END
|
|
21503
22135
|
WHERE project_id = ${input.projectId} AND id = ${input.artifactId}
|
|
21504
22136
|
RETURNING ${tx.unsafe(ARTIFACT_COLUMNS)}
|
|
21505
22137
|
`;
|
|
21506
|
-
return {
|
|
22138
|
+
return {
|
|
22139
|
+
artifact: parseArtifactRow(updated),
|
|
22140
|
+
version: parseArtifactVersionRow(version)
|
|
22141
|
+
};
|
|
22142
|
+
});
|
|
22143
|
+
}
|
|
22144
|
+
async listRestorePoints(input) {
|
|
22145
|
+
const rows = await this.sql`
|
|
22146
|
+
SELECT ${this.sql.unsafe(RESTORE_POINT_COLUMNS)} FROM artifact_restore_points
|
|
22147
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.id}
|
|
22148
|
+
ORDER BY revision DESC
|
|
22149
|
+
`;
|
|
22150
|
+
return rows.map(parseArtifactRestorePointRow);
|
|
22151
|
+
}
|
|
22152
|
+
async getRestorePoint(input) {
|
|
22153
|
+
const [row] = await this.sql`
|
|
22154
|
+
SELECT ${this.sql.unsafe(RESTORE_POINT_COLUMNS)} FROM artifact_restore_points
|
|
22155
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22156
|
+
AND revision = ${input.revision}
|
|
22157
|
+
`;
|
|
22158
|
+
return row ? parseArtifactRestorePointRow(row) : null;
|
|
22159
|
+
}
|
|
22160
|
+
async restoreTo(input) {
|
|
22161
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
22162
|
+
const current = await lockArtifact(tx, input.projectId, input.artifactId);
|
|
22163
|
+
if (!current)
|
|
22164
|
+
throw artifactStorageErrors.notFound(input.artifactId);
|
|
22165
|
+
const { artifact } = current;
|
|
22166
|
+
const [row] = await tx`
|
|
22167
|
+
SELECT ${tx.unsafe(RESTORE_POINT_COLUMNS)} FROM artifact_restore_points
|
|
22168
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22169
|
+
AND revision = ${input.revision}
|
|
22170
|
+
`;
|
|
22171
|
+
if (!row)
|
|
22172
|
+
throw artifactStorageErrors.restorePointNotFound(input.artifactId, input.revision);
|
|
22173
|
+
const point = parseArtifactRestorePointRow(row);
|
|
22174
|
+
const latest = await latestVersion(tx, artifact);
|
|
22175
|
+
assertBase(artifact, latest, input.baseRevision);
|
|
22176
|
+
await keepRestorePoint(tx, latest, { reason: "restore", createdAt: input.restoredAt });
|
|
22177
|
+
return writeLatest(tx, current, latest, {
|
|
22178
|
+
title: point.title,
|
|
22179
|
+
entry: point.entry,
|
|
22180
|
+
files: point.files,
|
|
22181
|
+
outline: point.outline,
|
|
22182
|
+
references: point.references,
|
|
22183
|
+
author: input.author,
|
|
22184
|
+
updatedAt: input.restoredAt
|
|
22185
|
+
}, input.document);
|
|
22186
|
+
});
|
|
22187
|
+
}
|
|
22188
|
+
async listTurnWrites(input) {
|
|
22189
|
+
const rows = await this.sql`
|
|
22190
|
+
SELECT ${this.sql.unsafe(TURN_WRITE_COLUMNS)} FROM artifact_turn_writes
|
|
22191
|
+
WHERE project_id = ${input.projectId} AND turn_run_id = ${input.turnRunId}
|
|
22192
|
+
ORDER BY artifact_id COLLATE "C"
|
|
22193
|
+
`;
|
|
22194
|
+
return rows.map(parseArtifactTurnWriteRow);
|
|
22195
|
+
}
|
|
22196
|
+
async undoTurnWrites(input) {
|
|
22197
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
22198
|
+
const { projectId, turnRunId } = input;
|
|
22199
|
+
await tx`
|
|
22200
|
+
SELECT id FROM artifacts
|
|
22201
|
+
WHERE project_id = ${projectId} AND id IN (
|
|
22202
|
+
SELECT artifact_id FROM artifact_turn_writes
|
|
22203
|
+
WHERE project_id = ${projectId} AND turn_run_id = ${turnRunId}
|
|
22204
|
+
)
|
|
22205
|
+
ORDER BY id
|
|
22206
|
+
FOR UPDATE
|
|
22207
|
+
`;
|
|
22208
|
+
const rows = await tx`
|
|
22209
|
+
SELECT w.artifact_id AS "artifactId", w.created, w.revision, w.writes,
|
|
22210
|
+
a.revision AS "currentRevision"
|
|
22211
|
+
FROM artifact_turn_writes w
|
|
22212
|
+
JOIN artifacts a ON a.project_id = w.project_id AND a.id = w.artifact_id
|
|
22213
|
+
WHERE w.project_id = ${projectId} AND w.turn_run_id = ${turnRunId}
|
|
22214
|
+
ORDER BY w.artifact_id COLLATE "C"
|
|
22215
|
+
`;
|
|
22216
|
+
const restored = [];
|
|
22217
|
+
const deleted = [];
|
|
22218
|
+
for (const row of rows) {
|
|
22219
|
+
const write = parseArtifactTurnWriteRow(row);
|
|
22220
|
+
const revision = Number(row.currentRevision);
|
|
22221
|
+
if (write.created) {
|
|
22222
|
+
if (!turnUndoable({ write, revision }))
|
|
22223
|
+
continue;
|
|
22224
|
+
const comments = await tx`
|
|
22225
|
+
SELECT ${tx.unsafe(COMMENT_COLUMNS)} FROM artifact_comments
|
|
22226
|
+
WHERE project_id = ${projectId} AND artifact_id = ${write.artifactId}
|
|
22227
|
+
`;
|
|
22228
|
+
if (othersCommented(comments.map(parseArtifactCommentRow), input))
|
|
22229
|
+
continue;
|
|
22230
|
+
await tx`DELETE FROM artifacts WHERE project_id = ${projectId} AND id = ${write.artifactId}`;
|
|
22231
|
+
deleted.push(write.artifactId);
|
|
22232
|
+
continue;
|
|
22233
|
+
}
|
|
22234
|
+
const undone = await undoTurn(tx, projectId, write, turnRunId, input.documents?.[write.artifactId]);
|
|
22235
|
+
if (undone)
|
|
22236
|
+
restored.push(undone);
|
|
22237
|
+
}
|
|
22238
|
+
if (restored.length > 0) {
|
|
22239
|
+
const undone = restored.map(({ artifact }) => artifact.id);
|
|
22240
|
+
await tx`
|
|
22241
|
+
DELETE FROM artifact_restore_points
|
|
22242
|
+
WHERE project_id = ${projectId} AND turn_run_id = ${turnRunId}
|
|
22243
|
+
AND artifact_id IN ${tx(undone)}
|
|
22244
|
+
`;
|
|
22245
|
+
await tx`
|
|
22246
|
+
DELETE FROM artifact_threads
|
|
22247
|
+
WHERE project_id = ${projectId} AND linked_by_run_id = ${turnRunId} AND NOT copied
|
|
22248
|
+
AND artifact_id IN ${tx(undone)}
|
|
22249
|
+
`;
|
|
22250
|
+
}
|
|
22251
|
+
await tx`
|
|
22252
|
+
DELETE FROM artifact_turn_writes
|
|
22253
|
+
WHERE project_id = ${projectId} AND turn_run_id = ${turnRunId}
|
|
22254
|
+
`;
|
|
22255
|
+
return { restored, deleted };
|
|
21507
22256
|
});
|
|
21508
22257
|
}
|
|
21509
22258
|
async get(input) {
|
|
@@ -21570,10 +22319,12 @@ class PgArtifactStorage {
|
|
|
21570
22319
|
const [artifact] = await tx`
|
|
21571
22320
|
INSERT INTO artifacts (
|
|
21572
22321
|
project_id, id, kind, title, owner_principal_type, owner_principal_id,
|
|
21573
|
-
latest_version, next_version, created_at, updated_at
|
|
22322
|
+
latest_version, next_version, revision, created_at, updated_at, document,
|
|
22323
|
+
document_revision
|
|
21574
22324
|
)
|
|
21575
22325
|
SELECT project_id, ${input.id}, kind, title, ${input.owner.type}, ${input.owner.id},
|
|
21576
|
-
latest_version, next_version, created_at, updated_at
|
|
22326
|
+
latest_version, next_version, revision, created_at, updated_at, document,
|
|
22327
|
+
document_revision
|
|
21577
22328
|
FROM artifacts WHERE project_id = ${input.projectId} AND id = ${input.sourceId}
|
|
21578
22329
|
ON CONFLICT (project_id, id) DO NOTHING
|
|
21579
22330
|
RETURNING ${tx.unsafe(ARTIFACT_COLUMNS)}
|
|
@@ -21582,14 +22333,32 @@ class PgArtifactStorage {
|
|
|
21582
22333
|
throw artifactStorageErrors.exists(input.id);
|
|
21583
22334
|
await tx`
|
|
21584
22335
|
INSERT INTO artifact_versions (
|
|
21585
|
-
project_id, artifact_id, version, title, entry, files, outline, refs,
|
|
21586
|
-
author_principal_type, author_principal_id, created_at
|
|
22336
|
+
project_id, artifact_id, version, revision, title, entry, files, outline, refs,
|
|
22337
|
+
author_principal_type, author_principal_id, created_at, updated_at, frozen_at,
|
|
22338
|
+
frozen_by_principal_type, frozen_by_principal_id
|
|
21587
22339
|
)
|
|
21588
|
-
SELECT project_id, ${input.id}, version, title, entry, files, outline, refs,
|
|
21589
|
-
author_principal_type, author_principal_id, created_at
|
|
22340
|
+
SELECT project_id, ${input.id}, version, revision, title, entry, files, outline, refs,
|
|
22341
|
+
author_principal_type, author_principal_id, created_at, updated_at, frozen_at,
|
|
22342
|
+
frozen_by_principal_type, frozen_by_principal_id
|
|
21590
22343
|
FROM artifact_versions
|
|
21591
22344
|
WHERE project_id = ${input.projectId} AND artifact_id = ${input.sourceId}
|
|
21592
22345
|
`;
|
|
22346
|
+
if (input.outlineOf) {
|
|
22347
|
+
const copied = await tx`
|
|
22348
|
+
SELECT ${tx.unsafe(VERSION_COLUMNS)} FROM artifact_versions
|
|
22349
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.id}
|
|
22350
|
+
`;
|
|
22351
|
+
for (const version of copied.map(parseArtifactVersionRow)) {
|
|
22352
|
+
const outline = input.outlineOf(version);
|
|
22353
|
+
if (!outline)
|
|
22354
|
+
continue;
|
|
22355
|
+
await tx`
|
|
22356
|
+
UPDATE artifact_versions SET outline = ${JSON.stringify(outline)}::text::jsonb
|
|
22357
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.id}
|
|
22358
|
+
AND version = ${version.version}
|
|
22359
|
+
`;
|
|
22360
|
+
}
|
|
22361
|
+
}
|
|
21593
22362
|
await tx`
|
|
21594
22363
|
INSERT INTO artifact_threads (project_id, artifact_id, thread_id, shared_at, copied)
|
|
21595
22364
|
VALUES (
|
|
@@ -21636,10 +22405,11 @@ class PgArtifactStorage {
|
|
|
21636
22405
|
async createFrameGrant(record) {
|
|
21637
22406
|
const rows = await this.sql`
|
|
21638
22407
|
INSERT INTO artifact_frame_grants
|
|
21639
|
-
(project_id, token_hash, artifact_id, version, targets, created_at,
|
|
22408
|
+
(project_id, token_hash, artifact_id, version, files_digest, targets, created_at,
|
|
22409
|
+
expires_at)
|
|
21640
22410
|
SELECT ${record.projectId}, ${record.tokenHash}, ${record.artifactId}, ${record.version},
|
|
21641
|
-
${JSON.stringify(record.targets)}::text::jsonb,
|
|
21642
|
-
${record.expiresAt}
|
|
22411
|
+
${record.filesDigest}, ${JSON.stringify(record.targets)}::text::jsonb,
|
|
22412
|
+
${record.createdAt}, ${record.expiresAt}
|
|
21643
22413
|
WHERE EXISTS (
|
|
21644
22414
|
SELECT 1 FROM artifacts WHERE project_id = ${record.projectId} AND id = ${record.artifactId}
|
|
21645
22415
|
)
|
|
@@ -21747,91 +22517,370 @@ class PgArtifactStorage {
|
|
|
21747
22517
|
`;
|
|
21748
22518
|
return rows.length;
|
|
21749
22519
|
}
|
|
21750
|
-
async
|
|
22520
|
+
async listBoardPreviews(input) {
|
|
22521
|
+
if (input.renderKeys.length === 0)
|
|
22522
|
+
return [];
|
|
21751
22523
|
const rows = await this.sql`
|
|
21752
|
-
|
|
21753
|
-
|
|
22524
|
+
SELECT ${this.sql.unsafe(BOARD_PREVIEW_COLUMNS)} FROM artifact_board_previews
|
|
22525
|
+
WHERE project_id = ${input.projectId}
|
|
22526
|
+
AND render_key = ANY(${this.sql.array([...new Set(input.renderKeys)])}::text[])
|
|
22527
|
+
ORDER BY render_key COLLATE "C", theme COLLATE "C", size COLLATE "C"
|
|
21754
22528
|
`;
|
|
21755
|
-
return rows.
|
|
22529
|
+
return rows.map(parseArtifactBoardPreviewRow);
|
|
21756
22530
|
}
|
|
21757
|
-
async
|
|
21758
|
-
|
|
21759
|
-
|
|
21760
|
-
|
|
21761
|
-
|
|
21762
|
-
|
|
21763
|
-
|
|
21764
|
-
|
|
21765
|
-
|
|
21766
|
-
|
|
21767
|
-
|
|
21768
|
-
const affected = locked.map((row) => row.id);
|
|
21769
|
-
if (affected.length === 0)
|
|
21770
|
-
return 0;
|
|
21771
|
-
const deleted = await tx`
|
|
21772
|
-
DELETE FROM artifact_versions
|
|
21773
|
-
WHERE project_id = ${input.projectId} AND origin_run_id = ${input.runId}
|
|
21774
|
-
AND artifact_id IN ${tx(affected)}
|
|
21775
|
-
RETURNING origin_thread_id
|
|
21776
|
-
`;
|
|
21777
|
-
const threadId = deleted[0]?.origin_thread_id;
|
|
21778
|
-
await tx`
|
|
21779
|
-
DELETE FROM artifact_threads t
|
|
21780
|
-
WHERE t.project_id = ${input.projectId} AND t.artifact_id IN ${tx(affected)}
|
|
21781
|
-
AND t.thread_id = ${threadId ?? null} AND NOT t.copied
|
|
21782
|
-
AND NOT EXISTS (
|
|
21783
|
-
SELECT 1 FROM artifact_versions v
|
|
21784
|
-
WHERE v.project_id = t.project_id AND v.artifact_id = t.artifact_id
|
|
21785
|
-
AND v.origin_thread_id = t.thread_id
|
|
21786
|
-
)
|
|
21787
|
-
`;
|
|
21788
|
-
await tx`
|
|
21789
|
-
DELETE FROM artifacts a
|
|
21790
|
-
WHERE a.project_id = ${input.projectId} AND a.id IN ${tx(affected)}
|
|
21791
|
-
AND NOT EXISTS (
|
|
21792
|
-
SELECT 1 FROM artifact_versions v
|
|
21793
|
-
WHERE v.project_id = a.project_id AND v.artifact_id = a.id
|
|
22531
|
+
async putBoardPreviews(input) {
|
|
22532
|
+
if (input.previews.length === 0)
|
|
22533
|
+
return;
|
|
22534
|
+
await runPgTransaction(this.sql, async (tx) => {
|
|
22535
|
+
for (const preview of input.previews) {
|
|
22536
|
+
await tx`
|
|
22537
|
+
INSERT INTO artifact_board_previews (
|
|
22538
|
+
project_id, render_key, theme, size, file, created_at
|
|
22539
|
+
) VALUES (
|
|
22540
|
+
${input.projectId}, ${preview.renderKey}, ${preview.theme}, ${preview.size},
|
|
22541
|
+
${JSON.stringify(preview.file)}::text::jsonb, ${preview.createdAt}
|
|
21794
22542
|
)
|
|
22543
|
+
ON CONFLICT (project_id, render_key, theme, size) DO NOTHING
|
|
22544
|
+
`;
|
|
22545
|
+
}
|
|
22546
|
+
});
|
|
22547
|
+
}
|
|
22548
|
+
async createComment(input) {
|
|
22549
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
22550
|
+
if (!await lockArtifact(tx, input.projectId, input.artifactId)) {
|
|
22551
|
+
throw artifactStorageErrors.notFound(input.artifactId);
|
|
22552
|
+
}
|
|
22553
|
+
if (input.parentId !== undefined) {
|
|
22554
|
+
const [parent] = await tx`
|
|
22555
|
+
SELECT 1 FROM artifact_comments
|
|
22556
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22557
|
+
AND id = ${input.parentId} AND parent_id IS NULL
|
|
22558
|
+
`;
|
|
22559
|
+
if (!parent)
|
|
22560
|
+
throw artifactStorageErrors.commentNotFound(input.artifactId, input.parentId);
|
|
22561
|
+
}
|
|
22562
|
+
const [counted] = await tx`
|
|
22563
|
+
SELECT COUNT(*)::int AS total,
|
|
22564
|
+
(COUNT(*) FILTER (WHERE author_principal_type = 'guest'))::int AS guests
|
|
22565
|
+
FROM artifact_comments
|
|
22566
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
21795
22567
|
`;
|
|
21796
|
-
|
|
21797
|
-
|
|
21798
|
-
|
|
21799
|
-
|
|
21800
|
-
|
|
21801
|
-
|
|
21802
|
-
|
|
21803
|
-
|
|
21804
|
-
|
|
21805
|
-
|
|
22568
|
+
if ((counted?.total ?? 0) >= ARTIFACT_COMMENT_LIMITS.maxComments) {
|
|
22569
|
+
throw artifactStorageErrors.commentsFull(input.artifactId);
|
|
22570
|
+
}
|
|
22571
|
+
if (input.author.type === "guest" && (counted?.guests ?? 0) >= ARTIFACT_COMMENT_LIMITS.maxGuestComments) {
|
|
22572
|
+
throw artifactStorageErrors.guestCommentsFull(input.artifactId);
|
|
22573
|
+
}
|
|
22574
|
+
const [row] = await tx`
|
|
22575
|
+
INSERT INTO artifact_comments (
|
|
22576
|
+
project_id, artifact_id, id, parent_id, anchor, body, author_principal_type,
|
|
22577
|
+
author_principal_id, author_name, created_at, origin_thread_id, origin_run_id
|
|
22578
|
+
) VALUES (
|
|
22579
|
+
${input.projectId}, ${input.artifactId}, ${input.id}, ${input.parentId ?? null},
|
|
22580
|
+
${JSON.stringify(commentAnchor(input.anchor))}::text::jsonb, ${input.body},
|
|
22581
|
+
${input.author.type}, ${input.author.id},
|
|
22582
|
+
${input.author.type === "guest" ? input.author.name : null}, ${input.createdAt},
|
|
22583
|
+
${input.origin?.threadId ?? null}, ${input.origin?.runId ?? null}
|
|
22584
|
+
)
|
|
22585
|
+
RETURNING ${tx.unsafe(COMMENT_COLUMNS)}
|
|
21806
22586
|
`;
|
|
21807
|
-
return
|
|
22587
|
+
return parseArtifactCommentRow(row);
|
|
21808
22588
|
});
|
|
21809
22589
|
}
|
|
22590
|
+
async getComment(input) {
|
|
22591
|
+
const [row] = await this.sql`
|
|
22592
|
+
SELECT ${this.sql.unsafe(COMMENT_COLUMNS)} FROM artifact_comments
|
|
22593
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22594
|
+
AND id = ${input.id}
|
|
22595
|
+
`;
|
|
22596
|
+
return row ? parseArtifactCommentRow(row) : null;
|
|
22597
|
+
}
|
|
22598
|
+
async listComments(input) {
|
|
22599
|
+
const rows = await this.sql`
|
|
22600
|
+
SELECT ${this.sql.unsafe(COMMENT_COLUMNS)} FROM artifact_comments
|
|
22601
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.id}
|
|
22602
|
+
ORDER BY created_at, seq
|
|
22603
|
+
`;
|
|
22604
|
+
return rows.map(parseArtifactCommentRow);
|
|
22605
|
+
}
|
|
22606
|
+
async resolveComment(input) {
|
|
22607
|
+
const [row] = await this.sql`
|
|
22608
|
+
UPDATE artifact_comments
|
|
22609
|
+
SET resolved_at = ${input.resolved?.at ?? null},
|
|
22610
|
+
resolved_by_principal_type = ${input.resolved?.by.type ?? null},
|
|
22611
|
+
resolved_by_principal_id = ${input.resolved?.by.id ?? null}
|
|
22612
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22613
|
+
AND id = ${input.id}
|
|
22614
|
+
RETURNING ${this.sql.unsafe(COMMENT_COLUMNS)}
|
|
22615
|
+
`;
|
|
22616
|
+
return row ? parseArtifactCommentRow(row) : null;
|
|
22617
|
+
}
|
|
22618
|
+
async deleteComment(input) {
|
|
22619
|
+
const rows = await this.sql`
|
|
22620
|
+
DELETE FROM artifact_comments
|
|
22621
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22622
|
+
AND id = ${input.id}
|
|
22623
|
+
RETURNING id
|
|
22624
|
+
`;
|
|
22625
|
+
return rows.length > 0;
|
|
22626
|
+
}
|
|
22627
|
+
async delete(input) {
|
|
22628
|
+
const rows = await this.sql`
|
|
22629
|
+
DELETE FROM artifacts WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
22630
|
+
RETURNING id
|
|
22631
|
+
`;
|
|
22632
|
+
return rows.length > 0;
|
|
22633
|
+
}
|
|
22634
|
+
}
|
|
22635
|
+
function commentAnchor(anchor) {
|
|
22636
|
+
return {
|
|
22637
|
+
...anchor.board === undefined ? {} : { board: anchor.board },
|
|
22638
|
+
...anchor.element === undefined ? {} : { element: anchor.element },
|
|
22639
|
+
...anchor.label === undefined ? {} : { label: anchor.label },
|
|
22640
|
+
...anchor.point === undefined ? {} : { point: { x: anchor.point.x, y: anchor.point.y } },
|
|
22641
|
+
revision: anchor.revision
|
|
22642
|
+
};
|
|
22643
|
+
}
|
|
22644
|
+
function documentJson(document) {
|
|
22645
|
+
return document ? JSON.stringify(document) : null;
|
|
21810
22646
|
}
|
|
21811
|
-
async function
|
|
22647
|
+
async function lockArtifact(sql, projectId, artifactId) {
|
|
22648
|
+
const [row] = await sql`
|
|
22649
|
+
SELECT ${sql.unsafe(ARTIFACT_COLUMNS)}, next_version AS "nextVersion" FROM artifacts
|
|
22650
|
+
WHERE project_id = ${projectId} AND id = ${artifactId}
|
|
22651
|
+
FOR UPDATE
|
|
22652
|
+
`;
|
|
22653
|
+
return row ? { artifact: parseArtifactRow(row), nextVersion: Number(row.nextVersion) } : null;
|
|
22654
|
+
}
|
|
22655
|
+
async function latestVersion(sql, artifact) {
|
|
22656
|
+
const [row] = await sql`
|
|
22657
|
+
SELECT ${sql.unsafe(VERSION_COLUMNS)} FROM artifact_versions
|
|
22658
|
+
WHERE project_id = ${artifact.projectId} AND artifact_id = ${artifact.id}
|
|
22659
|
+
AND version = ${artifact.latestVersion}
|
|
22660
|
+
`;
|
|
22661
|
+
if (!row)
|
|
22662
|
+
throw new Error(`[SixbPg] Artifact '${artifact.id}' has no latest version.`);
|
|
22663
|
+
return parseArtifactVersionRow(row);
|
|
22664
|
+
}
|
|
22665
|
+
function assertBase(artifact, latest, baseRevision) {
|
|
22666
|
+
if (baseRevision === undefined || baseRevision === artifact.revision)
|
|
22667
|
+
return;
|
|
22668
|
+
throw artifactStorageErrors.moved(artifact.id, baseRevision, {
|
|
22669
|
+
version: latest.version,
|
|
22670
|
+
revision: artifact.revision,
|
|
22671
|
+
author: latest.author,
|
|
22672
|
+
...latest.frozenBy ? { frozenBy: latest.frozenBy } : {}
|
|
22673
|
+
});
|
|
22674
|
+
}
|
|
22675
|
+
async function writeLatest(sql, { artifact, nextVersion }, latest, content, document) {
|
|
22676
|
+
const { projectId, id: artifactId } = artifact;
|
|
22677
|
+
const revision = artifact.revision + 1;
|
|
22678
|
+
const state = documentJson(document);
|
|
22679
|
+
if (latest.frozenAt) {
|
|
22680
|
+
const version = await insertVersion(sql, {
|
|
22681
|
+
projectId,
|
|
22682
|
+
artifactId,
|
|
22683
|
+
version: nextVersion,
|
|
22684
|
+
revision,
|
|
22685
|
+
content
|
|
22686
|
+
});
|
|
22687
|
+
const [row] = await sql`
|
|
22688
|
+
UPDATE artifacts
|
|
22689
|
+
SET title = ${content.title}, latest_version = ${version.version},
|
|
22690
|
+
next_version = ${version.version + 1}, revision = ${revision},
|
|
22691
|
+
updated_at = ${content.updatedAt},
|
|
22692
|
+
document = COALESCE(${state}::text::jsonb, document),
|
|
22693
|
+
document_revision = CASE WHEN ${state}::text IS NULL THEN document_revision
|
|
22694
|
+
ELSE ${revision}::int END
|
|
22695
|
+
WHERE project_id = ${projectId} AND id = ${artifactId}
|
|
22696
|
+
RETURNING ${sql.unsafe(ARTIFACT_COLUMNS)}
|
|
22697
|
+
`;
|
|
22698
|
+
return { artifact: parseArtifactRow(row), version };
|
|
22699
|
+
}
|
|
22700
|
+
const version = await replaceContent(sql, {
|
|
22701
|
+
projectId,
|
|
22702
|
+
artifactId,
|
|
22703
|
+
version: latest.version,
|
|
22704
|
+
revision,
|
|
22705
|
+
content
|
|
22706
|
+
});
|
|
22707
|
+
const [row] = await sql`
|
|
22708
|
+
UPDATE artifacts
|
|
22709
|
+
SET title = ${content.title}, revision = ${revision}, updated_at = ${content.updatedAt},
|
|
22710
|
+
document = COALESCE(${state}::text::jsonb, document),
|
|
22711
|
+
document_revision = CASE WHEN ${state}::text IS NULL THEN document_revision
|
|
22712
|
+
ELSE ${revision}::int END
|
|
22713
|
+
WHERE project_id = ${projectId} AND id = ${artifactId}
|
|
22714
|
+
RETURNING ${sql.unsafe(ARTIFACT_COLUMNS)}
|
|
22715
|
+
`;
|
|
22716
|
+
return { artifact: parseArtifactRow(row), version };
|
|
22717
|
+
}
|
|
22718
|
+
async function replaceContent(sql, input) {
|
|
22719
|
+
const { content } = input;
|
|
22720
|
+
const [row] = await sql`
|
|
22721
|
+
UPDATE artifact_versions
|
|
22722
|
+
SET revision = ${input.revision}, title = ${content.title}, entry = ${content.entry},
|
|
22723
|
+
files = ${JSON.stringify(content.files)}::text::jsonb,
|
|
22724
|
+
outline = ${JSON.stringify(content.outline)}::text::jsonb,
|
|
22725
|
+
refs = ${JSON.stringify(content.references)}::text::jsonb,
|
|
22726
|
+
author_principal_type = ${content.author.type},
|
|
22727
|
+
author_principal_id = ${content.author.id},
|
|
22728
|
+
origin_thread_id = ${content.origin?.threadId ?? null},
|
|
22729
|
+
origin_run_id = ${content.origin?.runId ?? null}, updated_at = ${content.updatedAt}
|
|
22730
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22731
|
+
AND version = ${input.version}
|
|
22732
|
+
RETURNING ${sql.unsafe(VERSION_COLUMNS)}
|
|
22733
|
+
`;
|
|
22734
|
+
return parseArtifactVersionRow(row);
|
|
22735
|
+
}
|
|
22736
|
+
async function insertVersion(sql, input) {
|
|
22737
|
+
const { content } = input;
|
|
21812
22738
|
const [row] = await sql`
|
|
21813
22739
|
INSERT INTO artifact_versions (
|
|
21814
|
-
project_id, artifact_id, version, title, entry, files, outline, refs,
|
|
21815
|
-
author_principal_type, author_principal_id, origin_thread_id, origin_run_id, created_at
|
|
22740
|
+
project_id, artifact_id, version, revision, title, entry, files, outline, refs,
|
|
22741
|
+
author_principal_type, author_principal_id, origin_thread_id, origin_run_id, created_at,
|
|
22742
|
+
updated_at
|
|
21816
22743
|
) VALUES (
|
|
21817
|
-
${projectId}, ${artifactId}, ${
|
|
21818
|
-
${
|
|
21819
|
-
${JSON.stringify(
|
|
21820
|
-
${JSON.stringify(
|
|
21821
|
-
${
|
|
21822
|
-
${
|
|
22744
|
+
${input.projectId}, ${input.artifactId}, ${input.version}, ${input.revision},
|
|
22745
|
+
${content.title}, ${content.entry},
|
|
22746
|
+
${JSON.stringify(content.files)}::text::jsonb,
|
|
22747
|
+
${JSON.stringify(content.outline)}::text::jsonb,
|
|
22748
|
+
${JSON.stringify(content.references)}::text::jsonb,
|
|
22749
|
+
${content.author.type}, ${content.author.id},
|
|
22750
|
+
${content.origin?.threadId ?? null}, ${content.origin?.runId ?? null},
|
|
22751
|
+
${content.updatedAt}, ${content.updatedAt}
|
|
21823
22752
|
)
|
|
21824
22753
|
RETURNING ${sql.unsafe(VERSION_COLUMNS)}
|
|
21825
22754
|
`;
|
|
21826
|
-
if (input.origin) {
|
|
21827
|
-
await linkThread(sql, projectId, artifactId, input.origin.threadId, input.createdAt);
|
|
21828
|
-
}
|
|
21829
22755
|
return parseArtifactVersionRow(row);
|
|
21830
22756
|
}
|
|
21831
|
-
async function
|
|
22757
|
+
async function keepRestorePoint(sql, latest, input) {
|
|
22758
|
+
const { projectId, artifactId } = latest;
|
|
21832
22759
|
await sql`
|
|
21833
|
-
INSERT INTO
|
|
21834
|
-
|
|
22760
|
+
INSERT INTO artifact_restore_points (
|
|
22761
|
+
project_id, artifact_id, revision, version, frozen, title, entry, files, outline, refs,
|
|
22762
|
+
author_principal_type, author_principal_id, origin_thread_id, origin_run_id, updated_at,
|
|
22763
|
+
reason, turn_run_id, created_at
|
|
22764
|
+
)
|
|
22765
|
+
SELECT project_id, artifact_id, revision, version, frozen_at IS NOT NULL, title, entry, files,
|
|
22766
|
+
outline, refs, author_principal_type, author_principal_id, origin_thread_id,
|
|
22767
|
+
origin_run_id, updated_at, ${input.reason}, ${input.turnRunId ?? null},
|
|
22768
|
+
${input.createdAt}
|
|
22769
|
+
FROM artifact_versions
|
|
22770
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId} AND version = ${latest.version}
|
|
22771
|
+
`;
|
|
22772
|
+
await sql`
|
|
22773
|
+
DELETE FROM artifact_restore_points
|
|
22774
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId} AND revision NOT IN (
|
|
22775
|
+
SELECT revision FROM artifact_restore_points
|
|
22776
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId}
|
|
22777
|
+
ORDER BY revision DESC LIMIT ${ARTIFACT_RESTORE_POINTS}
|
|
22778
|
+
)
|
|
22779
|
+
`;
|
|
22780
|
+
}
|
|
22781
|
+
async function turnWrote(sql, artifact, turnRunId) {
|
|
22782
|
+
const rows = await sql`
|
|
22783
|
+
SELECT 1 FROM artifact_turn_writes
|
|
22784
|
+
WHERE project_id = ${artifact.projectId} AND artifact_id = ${artifact.id}
|
|
22785
|
+
AND turn_run_id = ${turnRunId}
|
|
22786
|
+
`;
|
|
22787
|
+
return rows.length > 0;
|
|
22788
|
+
}
|
|
22789
|
+
async function recordTurnWrite(sql, input) {
|
|
22790
|
+
const { projectId, artifactId } = input;
|
|
22791
|
+
await sql`
|
|
22792
|
+
INSERT INTO artifact_turn_writes
|
|
22793
|
+
(project_id, artifact_id, turn_run_id, created, revision, writes)
|
|
22794
|
+
VALUES (${projectId}, ${artifactId}, ${input.turnRunId}, ${input.created}, ${input.revision}, 1)
|
|
22795
|
+
ON CONFLICT (project_id, artifact_id, turn_run_id) DO UPDATE SET
|
|
22796
|
+
revision = excluded.revision, writes = artifact_turn_writes.writes + 1
|
|
22797
|
+
`;
|
|
22798
|
+
await sql`
|
|
22799
|
+
DELETE FROM artifact_turn_writes
|
|
22800
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId} AND turn_run_id NOT IN (
|
|
22801
|
+
SELECT turn_run_id FROM artifact_turn_writes
|
|
22802
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId}
|
|
22803
|
+
ORDER BY revision DESC LIMIT ${ARTIFACT_RESTORE_POINTS}
|
|
22804
|
+
)
|
|
22805
|
+
`;
|
|
22806
|
+
}
|
|
22807
|
+
async function undoTurn(sql, projectId, write, turnRunId, document) {
|
|
22808
|
+
const { artifactId } = write;
|
|
22809
|
+
const [row] = await sql`
|
|
22810
|
+
SELECT ${sql.unsafe(RESTORE_POINT_COLUMNS)} FROM artifact_restore_points
|
|
22811
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId}
|
|
22812
|
+
AND turn_run_id = ${turnRunId}
|
|
22813
|
+
`;
|
|
22814
|
+
if (!row)
|
|
22815
|
+
return null;
|
|
22816
|
+
const point = parseArtifactRestorePointRow(row);
|
|
22817
|
+
const [current] = await sql`
|
|
22818
|
+
SELECT revision FROM artifacts WHERE project_id = ${projectId} AND id = ${artifactId}
|
|
22819
|
+
`;
|
|
22820
|
+
const [target] = await sql`
|
|
22821
|
+
SELECT frozen_at AS "frozenAt" FROM artifact_versions
|
|
22822
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId} AND version = ${point.version}
|
|
22823
|
+
`;
|
|
22824
|
+
if (!current || !target)
|
|
22825
|
+
return null;
|
|
22826
|
+
const untouched = turnUndoable({
|
|
22827
|
+
write,
|
|
22828
|
+
revision: Number(current.revision),
|
|
22829
|
+
point,
|
|
22830
|
+
target: { frozen: target.frozenAt !== null }
|
|
22831
|
+
});
|
|
22832
|
+
if (!untouched)
|
|
22833
|
+
return null;
|
|
22834
|
+
const revision = Number(current.revision) + 1;
|
|
22835
|
+
await sql`
|
|
22836
|
+
DELETE FROM artifact_versions
|
|
22837
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId} AND version > ${point.version}
|
|
22838
|
+
`;
|
|
22839
|
+
const at = { projectId, artifactId, version: point.version, revision };
|
|
22840
|
+
const undone = point.frozen ? await freshRevision(sql, at) : await replaceContent(sql, {
|
|
22841
|
+
...at,
|
|
22842
|
+
content: {
|
|
22843
|
+
title: point.title,
|
|
22844
|
+
entry: point.entry,
|
|
22845
|
+
files: point.files,
|
|
22846
|
+
outline: point.outline,
|
|
22847
|
+
references: point.references,
|
|
22848
|
+
author: point.author,
|
|
22849
|
+
...point.origin ? { origin: point.origin } : {},
|
|
22850
|
+
updatedAt: point.updatedAt
|
|
22851
|
+
}
|
|
22852
|
+
});
|
|
22853
|
+
const updatedAt = new Date(Math.max(undone.updatedAt.getTime(), undone.frozenAt?.getTime() ?? 0));
|
|
22854
|
+
const [artifact] = await sql`
|
|
22855
|
+
UPDATE artifacts
|
|
22856
|
+
SET title = ${undone.title}, latest_version = ${undone.version}, revision = ${revision},
|
|
22857
|
+
updated_at = ${updatedAt},
|
|
22858
|
+
document = COALESCE(${documentJson(document)}::text::jsonb, document),
|
|
22859
|
+
document_revision = CASE WHEN ${documentJson(document)}::text IS NULL THEN document_revision
|
|
22860
|
+
ELSE ${revision}::int END
|
|
22861
|
+
WHERE project_id = ${projectId} AND id = ${artifactId}
|
|
22862
|
+
RETURNING ${sql.unsafe(ARTIFACT_COLUMNS)}
|
|
22863
|
+
`;
|
|
22864
|
+
return { artifact: parseArtifactRow(artifact), version: undone };
|
|
22865
|
+
}
|
|
22866
|
+
async function freshRevision(sql, input) {
|
|
22867
|
+
const [row] = await sql`
|
|
22868
|
+
UPDATE artifact_versions SET revision = ${input.revision}
|
|
22869
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22870
|
+
AND version = ${input.version}
|
|
22871
|
+
RETURNING ${sql.unsafe(VERSION_COLUMNS)}
|
|
22872
|
+
`;
|
|
22873
|
+
return parseArtifactVersionRow(row);
|
|
22874
|
+
}
|
|
22875
|
+
async function shareIn(sql, projectId, artifactId, content, turnRunId) {
|
|
22876
|
+
if (!content.origin)
|
|
22877
|
+
return;
|
|
22878
|
+
await sql`
|
|
22879
|
+
INSERT INTO artifact_threads (project_id, artifact_id, thread_id, shared_at, linked_by_run_id)
|
|
22880
|
+
VALUES (
|
|
22881
|
+
${projectId}, ${artifactId}, ${content.origin.threadId}, ${content.updatedAt},
|
|
22882
|
+
${turnRunId ?? null}
|
|
22883
|
+
)
|
|
21835
22884
|
ON CONFLICT (project_id, artifact_id, thread_id) DO NOTHING
|
|
21836
22885
|
`;
|
|
21837
22886
|
}
|
|
@@ -21950,6 +22999,221 @@ function assertDirectTlsTarget(sql, options) {
|
|
|
21950
22999
|
return sql;
|
|
21951
23000
|
}
|
|
21952
23001
|
|
|
23002
|
+
// src/pg-dataset-ingest-storage.ts
|
|
23003
|
+
import {
|
|
23004
|
+
DurableDatasetIngestStorage,
|
|
23005
|
+
initialDatasetIngestState
|
|
23006
|
+
} from "@sixb/core/internal/dataset-ingest-storage-provider";
|
|
23007
|
+
class PgDatasetIngestStorage extends DurableDatasetIngestStorage {
|
|
23008
|
+
constructor(sql) {
|
|
23009
|
+
super(new PgDatasetIngestBackend(sql));
|
|
23010
|
+
}
|
|
23011
|
+
}
|
|
23012
|
+
|
|
23013
|
+
class PgDatasetIngestBackend {
|
|
23014
|
+
sql;
|
|
23015
|
+
constructor(sql) {
|
|
23016
|
+
this.sql = sql;
|
|
23017
|
+
}
|
|
23018
|
+
read(run) {
|
|
23019
|
+
return run(new PgDatasetIngestPersistence(this.sql));
|
|
23020
|
+
}
|
|
23021
|
+
transaction(run) {
|
|
23022
|
+
return runPgTransaction(this.sql, (tx) => run(new PgDatasetIngestPersistence(tx)));
|
|
23023
|
+
}
|
|
23024
|
+
}
|
|
23025
|
+
|
|
23026
|
+
class PgDatasetIngestPersistence {
|
|
23027
|
+
sql;
|
|
23028
|
+
constructor(sql) {
|
|
23029
|
+
this.sql = sql;
|
|
23030
|
+
}
|
|
23031
|
+
async now() {
|
|
23032
|
+
const [row] = await this.sql`SELECT clock_timestamp() AS now`;
|
|
23033
|
+
if (!row)
|
|
23034
|
+
throw new Error("[SixbPg] Database clock returned no row.");
|
|
23035
|
+
return new Date(row.now);
|
|
23036
|
+
}
|
|
23037
|
+
async lockState(projectId, datasetId) {
|
|
23038
|
+
await this.sql`
|
|
23039
|
+
INSERT INTO dataset_ingest_state (project_id, dataset_id)
|
|
23040
|
+
VALUES (${projectId}, ${datasetId})
|
|
23041
|
+
ON CONFLICT (project_id, dataset_id) DO NOTHING
|
|
23042
|
+
`;
|
|
23043
|
+
const [row] = await this.sql`
|
|
23044
|
+
SELECT dataset_id, request::text AS request, requested_at, lease_owner, lease_expires_at,
|
|
23045
|
+
flush_id, flush_started_at, failure_streak, last_flushed_at
|
|
23046
|
+
FROM dataset_ingest_state
|
|
23047
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId}
|
|
23048
|
+
FOR UPDATE
|
|
23049
|
+
`;
|
|
23050
|
+
return row ? stateFromRow(row) : initialDatasetIngestState(datasetId);
|
|
23051
|
+
}
|
|
23052
|
+
async writeState(projectId, state) {
|
|
23053
|
+
await this.sql`
|
|
23054
|
+
UPDATE dataset_ingest_state
|
|
23055
|
+
SET request = ${state.request},
|
|
23056
|
+
requested_at = ${state.requestedAt},
|
|
23057
|
+
lease_owner = ${state.leaseOwner},
|
|
23058
|
+
lease_expires_at = ${state.leaseExpiresAt},
|
|
23059
|
+
flush_id = ${state.flushId},
|
|
23060
|
+
flush_started_at = ${state.flushStartedAt},
|
|
23061
|
+
failure_streak = ${state.failureStreak},
|
|
23062
|
+
last_flushed_at = ${state.lastFlushedAt}
|
|
23063
|
+
WHERE project_id = ${projectId} AND dataset_id = ${state.datasetId}
|
|
23064
|
+
`;
|
|
23065
|
+
}
|
|
23066
|
+
async insertDelivery(projectId, record) {
|
|
23067
|
+
const [row] = await this.sql`
|
|
23068
|
+
INSERT INTO dataset_ingest_deliveries (
|
|
23069
|
+
project_id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
23070
|
+
definition_hash, changes
|
|
23071
|
+
) VALUES (
|
|
23072
|
+
${projectId}, ${record.datasetId}, ${record.executionId}, ${record.correlationId},
|
|
23073
|
+
${record.receivedAt}, ${record.changeCount}, ${record.definitionHash}, ${record.changes}
|
|
23074
|
+
)
|
|
23075
|
+
RETURNING id::text AS id
|
|
23076
|
+
`;
|
|
23077
|
+
if (!row)
|
|
23078
|
+
throw new Error("[SixbPg] Dataset ingest delivery insert returned no id.");
|
|
23079
|
+
return row.id;
|
|
23080
|
+
}
|
|
23081
|
+
async listUntaken(projectId, datasetId, limit) {
|
|
23082
|
+
const rows = await this.sql`
|
|
23083
|
+
-- The id is read as text: ORDER BY names the table's column, not that alias.
|
|
23084
|
+
SELECT id::text AS id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
23085
|
+
definition_hash
|
|
23086
|
+
FROM dataset_ingest_deliveries
|
|
23087
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id IS NULL
|
|
23088
|
+
AND rejected_at IS NULL
|
|
23089
|
+
ORDER BY dataset_ingest_deliveries.id
|
|
23090
|
+
LIMIT ${limit}
|
|
23091
|
+
`;
|
|
23092
|
+
return rows.map(deliveryFromRow);
|
|
23093
|
+
}
|
|
23094
|
+
async take(projectId, datasetId, deliveryIds, flushId) {
|
|
23095
|
+
await this.sql`
|
|
23096
|
+
UPDATE dataset_ingest_deliveries
|
|
23097
|
+
SET flush_id = ${flushId}
|
|
23098
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId}
|
|
23099
|
+
AND id = ANY(${this.sql.array([...deliveryIds])}::bigint[])
|
|
23100
|
+
`;
|
|
23101
|
+
}
|
|
23102
|
+
async listFlush(projectId, datasetId, flushId) {
|
|
23103
|
+
const rows = await this.sql`
|
|
23104
|
+
SELECT id::text AS id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
23105
|
+
definition_hash
|
|
23106
|
+
FROM dataset_ingest_deliveries
|
|
23107
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id = ${flushId}
|
|
23108
|
+
ORDER BY dataset_ingest_deliveries.id
|
|
23109
|
+
`;
|
|
23110
|
+
return rows.map(deliveryFromRow);
|
|
23111
|
+
}
|
|
23112
|
+
async readFlush(projectId, datasetId, flushId, page) {
|
|
23113
|
+
const rows = await this.sql`
|
|
23114
|
+
SELECT id::text AS id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
23115
|
+
definition_hash, changes
|
|
23116
|
+
FROM dataset_ingest_deliveries
|
|
23117
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id = ${flushId}
|
|
23118
|
+
AND id > ${page.afterId ?? "0"}::bigint
|
|
23119
|
+
ORDER BY dataset_ingest_deliveries.id
|
|
23120
|
+
LIMIT ${page.limit}
|
|
23121
|
+
`;
|
|
23122
|
+
return rows.map((row) => ({ delivery: deliveryFromRow(row), changes: row.changes }));
|
|
23123
|
+
}
|
|
23124
|
+
async deleteFlush(projectId, datasetId, flushId) {
|
|
23125
|
+
await this.sql`
|
|
23126
|
+
DELETE FROM dataset_ingest_deliveries
|
|
23127
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id = ${flushId}
|
|
23128
|
+
`;
|
|
23129
|
+
}
|
|
23130
|
+
async rejectDelivery(projectId, datasetId, flushId, deliveryId, rejection) {
|
|
23131
|
+
await this.sql`
|
|
23132
|
+
UPDATE dataset_ingest_deliveries
|
|
23133
|
+
SET flush_id = NULL, rejected_at = ${rejection.rejectedAt}, rejection = ${rejection.reason}
|
|
23134
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id = ${flushId}
|
|
23135
|
+
AND id = ${deliveryId}::bigint
|
|
23136
|
+
`;
|
|
23137
|
+
}
|
|
23138
|
+
async listRejected(projectId, datasetId, limit) {
|
|
23139
|
+
const rows = await this.sql`
|
|
23140
|
+
SELECT id::text AS id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
23141
|
+
definition_hash, changes, rejected_at, rejection
|
|
23142
|
+
FROM dataset_ingest_deliveries
|
|
23143
|
+
WHERE project_id = ${projectId} AND rejected_at IS NOT NULL
|
|
23144
|
+
AND (${datasetId === undefined} OR dataset_id = ${datasetId ?? ""})
|
|
23145
|
+
ORDER BY rejected_at DESC, dataset_ingest_deliveries.id DESC
|
|
23146
|
+
LIMIT ${limit}
|
|
23147
|
+
`;
|
|
23148
|
+
return rows.map((row) => ({
|
|
23149
|
+
delivery: deliveryFromRow(row),
|
|
23150
|
+
rejectedAt: new Date(row.rejected_at),
|
|
23151
|
+
reason: row.rejection,
|
|
23152
|
+
changes: row.changes
|
|
23153
|
+
}));
|
|
23154
|
+
}
|
|
23155
|
+
async deleteRejected(projectId, before) {
|
|
23156
|
+
await this.sql`
|
|
23157
|
+
DELETE FROM dataset_ingest_deliveries
|
|
23158
|
+
WHERE project_id = ${projectId} AND rejected_at < ${before}
|
|
23159
|
+
`;
|
|
23160
|
+
}
|
|
23161
|
+
async listStates(projectId, datasetIds) {
|
|
23162
|
+
const rows = await this.sql`
|
|
23163
|
+
SELECT dataset_id, request::text AS request, requested_at, lease_owner, lease_expires_at,
|
|
23164
|
+
flush_id, flush_started_at, failure_streak, last_flushed_at
|
|
23165
|
+
FROM dataset_ingest_state
|
|
23166
|
+
WHERE project_id = ${projectId}
|
|
23167
|
+
AND (${datasetIds === undefined} OR dataset_id = ANY(${this.sql.array([...datasetIds ?? []])}::text[]))
|
|
23168
|
+
`;
|
|
23169
|
+
return rows.map(stateFromRow);
|
|
23170
|
+
}
|
|
23171
|
+
async countPending(projectId, datasetIds) {
|
|
23172
|
+
const rows = await this.sql`
|
|
23173
|
+
SELECT dataset_id,
|
|
23174
|
+
count(*) FILTER (WHERE rejected_at IS NULL)::text AS deliveries,
|
|
23175
|
+
coalesce(sum(change_count) FILTER (WHERE rejected_at IS NULL), 0)::text AS changes,
|
|
23176
|
+
min(received_at) FILTER (WHERE rejected_at IS NULL) AS oldest,
|
|
23177
|
+
count(*) FILTER (WHERE rejected_at IS NOT NULL)::text AS rejected
|
|
23178
|
+
FROM dataset_ingest_deliveries
|
|
23179
|
+
WHERE project_id = ${projectId}
|
|
23180
|
+
AND (${datasetIds === undefined} OR dataset_id = ANY(${this.sql.array([...datasetIds ?? []])}::text[]))
|
|
23181
|
+
GROUP BY dataset_id
|
|
23182
|
+
`;
|
|
23183
|
+
return rows.map((row) => ({
|
|
23184
|
+
datasetId: row.dataset_id,
|
|
23185
|
+
pendingDeliveries: Number(row.deliveries),
|
|
23186
|
+
pendingChanges: Number(row.changes),
|
|
23187
|
+
oldestReceivedAt: row.oldest === null ? null : new Date(row.oldest),
|
|
23188
|
+
rejectedDeliveries: Number(row.rejected)
|
|
23189
|
+
}));
|
|
23190
|
+
}
|
|
23191
|
+
}
|
|
23192
|
+
function stateFromRow(row) {
|
|
23193
|
+
return {
|
|
23194
|
+
datasetId: row.dataset_id,
|
|
23195
|
+
request: Number(row.request),
|
|
23196
|
+
requestedAt: row.requested_at === null ? null : new Date(row.requested_at),
|
|
23197
|
+
leaseOwner: row.lease_owner,
|
|
23198
|
+
leaseExpiresAt: row.lease_expires_at === null ? null : new Date(row.lease_expires_at),
|
|
23199
|
+
flushId: row.flush_id,
|
|
23200
|
+
flushStartedAt: row.flush_started_at === null ? null : new Date(row.flush_started_at),
|
|
23201
|
+
failureStreak: row.failure_streak,
|
|
23202
|
+
lastFlushedAt: row.last_flushed_at === null ? null : new Date(row.last_flushed_at)
|
|
23203
|
+
};
|
|
23204
|
+
}
|
|
23205
|
+
function deliveryFromRow(row) {
|
|
23206
|
+
return {
|
|
23207
|
+
id: row.id,
|
|
23208
|
+
datasetId: row.dataset_id,
|
|
23209
|
+
executionId: row.execution_id,
|
|
23210
|
+
correlationId: row.correlation_id,
|
|
23211
|
+
receivedAt: new Date(row.received_at),
|
|
23212
|
+
changeCount: row.change_count,
|
|
23213
|
+
definitionHash: row.definition_hash
|
|
23214
|
+
};
|
|
23215
|
+
}
|
|
23216
|
+
|
|
21953
23217
|
// src/pg-file-download-grant-storage.ts
|
|
21954
23218
|
import { parseFileDownloadGrantRow } from "@sixb/core/internal/storage";
|
|
21955
23219
|
|
|
@@ -22138,12 +23402,88 @@ function jsonText(value) {
|
|
|
22138
23402
|
return value === undefined ? null : JSON.stringify(value);
|
|
22139
23403
|
}
|
|
22140
23404
|
|
|
23405
|
+
// src/pg-lake-retention-run-storage.ts
|
|
23406
|
+
class PgLakeRetentionRunStorage {
|
|
23407
|
+
sql;
|
|
23408
|
+
constructor(sql) {
|
|
23409
|
+
this.sql = sql;
|
|
23410
|
+
}
|
|
23411
|
+
async record(run) {
|
|
23412
|
+
await this.sql`
|
|
23413
|
+
INSERT INTO lake_retention_runs (project_id, started_at, finished_at, failed)
|
|
23414
|
+
VALUES (${run.projectId}, ${run.startedAt}, ${run.finishedAt}, ${run.failed})
|
|
23415
|
+
ON CONFLICT (project_id) DO UPDATE SET
|
|
23416
|
+
started_at = EXCLUDED.started_at,
|
|
23417
|
+
finished_at = EXCLUDED.finished_at,
|
|
23418
|
+
failed = EXCLUDED.failed
|
|
23419
|
+
`;
|
|
23420
|
+
}
|
|
23421
|
+
async getLast(input) {
|
|
23422
|
+
const [row] = await this.sql`
|
|
23423
|
+
SELECT project_id, started_at, finished_at, failed
|
|
23424
|
+
FROM lake_retention_runs
|
|
23425
|
+
WHERE project_id = ${input.projectId}
|
|
23426
|
+
`;
|
|
23427
|
+
if (!row)
|
|
23428
|
+
return null;
|
|
23429
|
+
return {
|
|
23430
|
+
projectId: row.project_id,
|
|
23431
|
+
startedAt: new Date(row.started_at),
|
|
23432
|
+
finishedAt: row.finished_at ? new Date(row.finished_at) : null,
|
|
23433
|
+
failed: row.failed
|
|
23434
|
+
};
|
|
23435
|
+
}
|
|
23436
|
+
}
|
|
23437
|
+
|
|
23438
|
+
// src/pg-maintenance-heartbeat-storage.ts
|
|
23439
|
+
import {
|
|
23440
|
+
parseMaintenanceHeartbeatRow,
|
|
23441
|
+
serializeMaintenancePasses
|
|
23442
|
+
} from "@sixb/core/internal/storage";
|
|
23443
|
+
|
|
23444
|
+
class PgMaintenanceHeartbeatStorage {
|
|
23445
|
+
sql;
|
|
23446
|
+
constructor(sql) {
|
|
23447
|
+
this.sql = sql;
|
|
23448
|
+
}
|
|
23449
|
+
async record(heartbeat) {
|
|
23450
|
+
const passes = serializeMaintenancePasses(heartbeat.passes);
|
|
23451
|
+
await this.sql`
|
|
23452
|
+
INSERT INTO maintenance_heartbeats (project_id, owner, beat_at, every_ms, passes)
|
|
23453
|
+
VALUES (
|
|
23454
|
+
${heartbeat.projectId}, ${heartbeat.owner}, ${heartbeat.beatAt}, ${heartbeat.everyMs},
|
|
23455
|
+
${passes}::text::jsonb
|
|
23456
|
+
)
|
|
23457
|
+
ON CONFLICT (project_id, owner) DO UPDATE SET
|
|
23458
|
+
beat_at = EXCLUDED.beat_at,
|
|
23459
|
+
every_ms = EXCLUDED.every_ms,
|
|
23460
|
+
passes = EXCLUDED.passes
|
|
23461
|
+
`;
|
|
23462
|
+
}
|
|
23463
|
+
async list(input) {
|
|
23464
|
+
const rows = await this.sql`
|
|
23465
|
+
SELECT project_id, owner, beat_at, every_ms, passes
|
|
23466
|
+
FROM maintenance_heartbeats
|
|
23467
|
+
WHERE project_id = ${input.projectId}
|
|
23468
|
+
`;
|
|
23469
|
+
return rows.map((row) => parseMaintenanceHeartbeatRow({
|
|
23470
|
+
projectId: row.project_id,
|
|
23471
|
+
owner: row.owner,
|
|
23472
|
+
beatAt: row.beat_at,
|
|
23473
|
+
everyMs: row.every_ms,
|
|
23474
|
+
passes: row.passes
|
|
23475
|
+
}));
|
|
23476
|
+
}
|
|
23477
|
+
}
|
|
23478
|
+
|
|
22141
23479
|
// src/pg-pipeline-run-storage.ts
|
|
22142
23480
|
import { parseSixbFailure as parseSixbFailure4, serializeSixbFailure as serializeSixbFailure3 } from "@sixb/core/internal/errors";
|
|
22143
23481
|
import { assertPipelineRunExecution } from "@sixb/core/internal/pipeline-run-storage-provider";
|
|
22144
23482
|
import {
|
|
22145
23483
|
canFinishPipelineRun,
|
|
22146
|
-
canRequeuePipelineRunAfterEnqueueFailure
|
|
23484
|
+
canRequeuePipelineRunAfterEnqueueFailure,
|
|
23485
|
+
holdsRun,
|
|
23486
|
+
mayFinishRun
|
|
22147
23487
|
} from "@sixb/core/internal/storage";
|
|
22148
23488
|
import { PIPELINE_RUN_FAILURE_CODES, PipelineRunError } from "@sixb/core/storage";
|
|
22149
23489
|
|
|
@@ -22174,7 +23514,99 @@ async function queryLatestRunsByOwnerId(input) {
|
|
|
22174
23514
|
});
|
|
22175
23515
|
}
|
|
22176
23516
|
|
|
23517
|
+
// src/run-claim-sql.ts
|
|
23518
|
+
import { laterLease, planRunClaim } from "@sixb/core/internal/storage";
|
|
23519
|
+
function pgRunExecutionFields(row) {
|
|
23520
|
+
return {
|
|
23521
|
+
attempt: Number(row.attempt),
|
|
23522
|
+
execution: row.execution_token && row.execution_queue_lease_expires_at ? {
|
|
23523
|
+
token: row.execution_token,
|
|
23524
|
+
queueLeaseExpiresAt: new Date(row.execution_queue_lease_expires_at)
|
|
23525
|
+
} : undefined
|
|
23526
|
+
};
|
|
23527
|
+
}
|
|
23528
|
+
async function lockPgRun(tx, table, projectId, id) {
|
|
23529
|
+
const [row] = await tx.unsafe(`SELECT ${table.returning} FROM ${table.name} WHERE project_id = $1 AND id = $2 FOR UPDATE`, [projectId, id]);
|
|
23530
|
+
if (!row)
|
|
23531
|
+
throw table.notFound(projectId, id);
|
|
23532
|
+
return table.toRun(row);
|
|
23533
|
+
}
|
|
23534
|
+
async function claimPgRun(tx, table, input, sibling) {
|
|
23535
|
+
const existing = await lockPgRun(tx, table, input.projectId, input.id);
|
|
23536
|
+
const plan = planRunClaim(existing, input, sibling);
|
|
23537
|
+
if (!plan.take)
|
|
23538
|
+
return plan.claim;
|
|
23539
|
+
const [updated] = await tx.unsafe(`UPDATE ${table.name}
|
|
23540
|
+
SET status = 'running', started_at = $1, attempt = $2, execution_token = $3,
|
|
23541
|
+
execution_queue_lease_expires_at = $4, error = NULL
|
|
23542
|
+
WHERE project_id = $5 AND id = $6
|
|
23543
|
+
RETURNING ${table.returning}`, [
|
|
23544
|
+
plan.startedAt,
|
|
23545
|
+
plan.attempt,
|
|
23546
|
+
plan.execution.token,
|
|
23547
|
+
plan.execution.queueLeaseExpiresAt,
|
|
23548
|
+
input.projectId,
|
|
23549
|
+
input.id
|
|
23550
|
+
]);
|
|
23551
|
+
if (!updated)
|
|
23552
|
+
throw table.notFound(input.projectId, input.id);
|
|
23553
|
+
return { outcome: plan.outcome, run: table.toRun(updated) };
|
|
23554
|
+
}
|
|
23555
|
+
async function lockHeldPgRun(tx, table, input) {
|
|
23556
|
+
const run = await lockPgRun(tx, table, input.projectId, input.id);
|
|
23557
|
+
if (run.status !== "running" || run.execution?.token !== input.executionToken) {
|
|
23558
|
+
throw table.staleExecution(input.id);
|
|
23559
|
+
}
|
|
23560
|
+
return run;
|
|
23561
|
+
}
|
|
23562
|
+
async function confirmPgRunExecution(tx, table, input) {
|
|
23563
|
+
const existing = await lockHeldPgRun(tx, table, input);
|
|
23564
|
+
const [updated] = await tx.unsafe(`UPDATE ${table.name} SET execution_queue_lease_expires_at = $1
|
|
23565
|
+
WHERE project_id = $2 AND id = $3
|
|
23566
|
+
RETURNING ${table.returning}`, [
|
|
23567
|
+
laterLease(existing.execution?.queueLeaseExpiresAt, input.queueLeaseExpiresAt),
|
|
23568
|
+
input.projectId,
|
|
23569
|
+
input.id
|
|
23570
|
+
]);
|
|
23571
|
+
if (!updated)
|
|
23572
|
+
throw table.notFound(input.projectId, input.id);
|
|
23573
|
+
return table.toRun(updated);
|
|
23574
|
+
}
|
|
23575
|
+
async function listLostPgRuns(sql, table, input) {
|
|
23576
|
+
const rows = await sql.unsafe(`SELECT ${table.returning} FROM ${table.name}
|
|
23577
|
+
WHERE project_id = $1
|
|
23578
|
+
AND status IN ('queued', 'running')
|
|
23579
|
+
AND (
|
|
23580
|
+
(status = 'queued' AND queued_at <= $2)
|
|
23581
|
+
OR (status = 'running' AND execution_queue_lease_expires_at <= $3)
|
|
23582
|
+
OR (status = 'running' AND execution_queue_lease_expires_at IS NULL AND started_at <= $4)
|
|
23583
|
+
)
|
|
23584
|
+
ORDER BY status = 'running' DESC,
|
|
23585
|
+
COALESCE(execution_queue_lease_expires_at, started_at, queued_at)
|
|
23586
|
+
LIMIT $5`, [input.projectId, input.queuedBefore, input.lapsedBy, input.orphanedBefore, input.limit]);
|
|
23587
|
+
return rows.map(table.toRun);
|
|
23588
|
+
}
|
|
23589
|
+
async function releasePgRun(sql, table, input) {
|
|
23590
|
+
await sql.unsafe(`UPDATE ${table.name}
|
|
23591
|
+
SET execution_queue_lease_expires_at = $1, attempt = attempt - $2
|
|
23592
|
+
WHERE project_id = $3 AND id = $4 AND status = 'running' AND execution_token = $5`, [
|
|
23593
|
+
input.releasedAt ?? new Date,
|
|
23594
|
+
input.spentAttempt ? 0 : 1,
|
|
23595
|
+
input.projectId,
|
|
23596
|
+
input.id,
|
|
23597
|
+
input.executionToken
|
|
23598
|
+
]);
|
|
23599
|
+
}
|
|
23600
|
+
|
|
22177
23601
|
// src/pg-pipeline-run-storage.ts
|
|
23602
|
+
var PIPELINE_RUNS = {
|
|
23603
|
+
name: "pipeline_runs",
|
|
23604
|
+
returning: "*",
|
|
23605
|
+
toRun: rowToPipelineRunRecord,
|
|
23606
|
+
notFound: (projectId, id) => new PipelineRunError(`[SixbPg] Pipeline run '${id}' not found for project '${projectId}'.`),
|
|
23607
|
+
staleExecution
|
|
23608
|
+
};
|
|
23609
|
+
|
|
22178
23610
|
class PgPipelineRunStorage {
|
|
22179
23611
|
sql;
|
|
22180
23612
|
executions;
|
|
@@ -22232,6 +23664,7 @@ class PgPipelineRunStorage {
|
|
|
22232
23664
|
const [updated] = await tx`
|
|
22233
23665
|
UPDATE pipeline_runs
|
|
22234
23666
|
SET status = ${"queued"}, queued_at = ${input.queuedAt ?? new Date},
|
|
23667
|
+
attempt = ${0}, execution_token = ${null}, execution_queue_lease_expires_at = ${null},
|
|
22235
23668
|
started_at = ${null}, finished_at = ${null}, output_dataset_id = ${null},
|
|
22236
23669
|
output_version_id = ${null}, error = ${null}
|
|
22237
23670
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
@@ -22240,38 +23673,32 @@ class PgPipelineRunStorage {
|
|
|
22240
23673
|
return rowToPipelineRunRecord(updated);
|
|
22241
23674
|
});
|
|
22242
23675
|
}
|
|
22243
|
-
async
|
|
22244
|
-
|
|
22245
|
-
|
|
22246
|
-
|
|
22247
|
-
|
|
22248
|
-
|
|
22249
|
-
|
|
22250
|
-
|
|
22251
|
-
|
|
22252
|
-
|
|
22253
|
-
|
|
22254
|
-
if (!existing) {
|
|
22255
|
-
throw new PipelineRunError(`[SixbPg] Pipeline run '${input.id}' not found for project '${input.projectId}'.`);
|
|
22256
|
-
}
|
|
22257
|
-
throw new PipelineRunError(`[SixbPg] Pipeline run '${input.id}' cannot start from status '${existing.status}'.`);
|
|
23676
|
+
async startOrReclaim(input) {
|
|
23677
|
+
return runPgTransaction(this.sql, (tx) => claimPgRun(tx, PIPELINE_RUNS, input));
|
|
23678
|
+
}
|
|
23679
|
+
async confirmExecutionOwnership(input) {
|
|
23680
|
+
return runPgTransaction(this.sql, (tx) => confirmPgRunExecution(tx, PIPELINE_RUNS, input));
|
|
23681
|
+
}
|
|
23682
|
+
async listLost(input) {
|
|
23683
|
+
return listLostPgRuns(this.sql, PIPELINE_RUNS, input);
|
|
23684
|
+
}
|
|
23685
|
+
async releaseExecution(input) {
|
|
23686
|
+
await releasePgRun(this.sql, PIPELINE_RUNS, input);
|
|
22258
23687
|
}
|
|
22259
23688
|
async finish(input) {
|
|
22260
23689
|
return runPgTransaction(this.sql, async (tx) => {
|
|
22261
|
-
const
|
|
22262
|
-
SELECT * FROM pipeline_runs
|
|
22263
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
22264
|
-
`;
|
|
22265
|
-
if (!existing) {
|
|
22266
|
-
throw new PipelineRunError(`[SixbPg] Pipeline run '${input.id}' not found for project '${input.projectId}'.`);
|
|
22267
|
-
}
|
|
23690
|
+
const existing = await lockPgRun(tx, PIPELINE_RUNS, input.projectId, input.id);
|
|
22268
23691
|
if (!canFinishPipelineRun(existing.status, input)) {
|
|
22269
23692
|
throw new PipelineRunError(`[SixbPg] Pipeline run '${input.id}' cannot finish from status '${existing.status}'.`);
|
|
22270
23693
|
}
|
|
23694
|
+
if (!mayFinishRun(existing, input))
|
|
23695
|
+
throw staleExecution(input.id);
|
|
22271
23696
|
const [updated] = input.status === "succeeded" ? await tx`
|
|
22272
23697
|
UPDATE pipeline_runs
|
|
22273
23698
|
SET
|
|
22274
23699
|
status = ${input.status},
|
|
23700
|
+
execution_token = ${null},
|
|
23701
|
+
execution_queue_lease_expires_at = ${null},
|
|
22275
23702
|
finished_at = ${input.finishedAt ?? new Date},
|
|
22276
23703
|
output_dataset_id = ${input.output?.datasetId ?? null},
|
|
22277
23704
|
output_version_id = ${input.output?.versionId ?? null},
|
|
@@ -22282,6 +23709,8 @@ class PgPipelineRunStorage {
|
|
|
22282
23709
|
UPDATE pipeline_runs
|
|
22283
23710
|
SET
|
|
22284
23711
|
status = ${input.status},
|
|
23712
|
+
execution_token = ${null},
|
|
23713
|
+
execution_queue_lease_expires_at = ${null},
|
|
22285
23714
|
finished_at = ${input.finishedAt ?? new Date},
|
|
22286
23715
|
output_dataset_id = ${null},
|
|
22287
23716
|
output_version_id = ${null},
|
|
@@ -22294,19 +23723,23 @@ class PgPipelineRunStorage {
|
|
|
22294
23723
|
}
|
|
22295
23724
|
async startStep(input) {
|
|
22296
23725
|
return runPgTransaction(this.sql, async (tx) => {
|
|
22297
|
-
const
|
|
22298
|
-
SELECT * FROM pipeline_runs
|
|
22299
|
-
WHERE project_id = ${input.projectId} AND id = ${input.pipelineRunId}
|
|
22300
|
-
`;
|
|
22301
|
-
if (!pipelineRun) {
|
|
22302
|
-
throw new PipelineRunError(`[SixbPg] Pipeline run '${input.pipelineRunId}' not found for project '${input.projectId}'.`);
|
|
22303
|
-
}
|
|
23726
|
+
const pipelineRun = await lockPgRun(tx, PIPELINE_RUNS, input.projectId, input.pipelineRunId);
|
|
22304
23727
|
if (pipelineRun.status !== "running") {
|
|
22305
23728
|
throw new PipelineRunError(`[SixbPg] Pipeline run '${input.pipelineRunId}' for project '${input.projectId}' is already terminal.`);
|
|
22306
23729
|
}
|
|
22307
|
-
if (pipelineRun
|
|
22308
|
-
throw
|
|
23730
|
+
if (!holdsRun(pipelineRun, input.executionToken))
|
|
23731
|
+
throw staleExecution(input.pipelineRunId);
|
|
23732
|
+
if (pipelineRun.pipelineId !== input.pipelineId) {
|
|
23733
|
+
throw new PipelineRunError(`[SixbPg] Pipeline step run '${input.id}' pipeline '${input.pipelineId}' does not match pipeline run '${input.pipelineRunId}' pipeline '${pipelineRun.pipelineId}'.`);
|
|
22309
23734
|
}
|
|
23735
|
+
const [restarted] = await tx`
|
|
23736
|
+
UPDATE pipeline_step_runs
|
|
23737
|
+
SET inputs = ${JSON.stringify(input.inputs)}::text::jsonb
|
|
23738
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"running"}
|
|
23739
|
+
RETURNING *
|
|
23740
|
+
`;
|
|
23741
|
+
if (restarted)
|
|
23742
|
+
return rowToPipelineStepRunRecord(restarted);
|
|
22310
23743
|
try {
|
|
22311
23744
|
const [row] = await tx`
|
|
22312
23745
|
INSERT INTO pipeline_step_runs (
|
|
@@ -22356,6 +23789,10 @@ class PgPipelineRunStorage {
|
|
|
22356
23789
|
if (existing.status !== "running") {
|
|
22357
23790
|
throw new PipelineRunError(`[SixbPg] Pipeline step run '${input.id}' for project '${input.projectId}' is already terminal.`);
|
|
22358
23791
|
}
|
|
23792
|
+
const pipelineRun = await lockPgRun(tx, PIPELINE_RUNS, input.projectId, existing.pipeline_run_id);
|
|
23793
|
+
if (!holdsRun(pipelineRun, input.executionToken)) {
|
|
23794
|
+
throw staleExecution(existing.pipeline_run_id);
|
|
23795
|
+
}
|
|
22359
23796
|
if (input.status === "succeeded" && input.output.datasetId !== existing.dataset_id) {
|
|
22360
23797
|
throw new PipelineRunError(`[SixbPg] Pipeline step run '${input.id}' output dataset '${input.output.datasetId}' does not match '${existing.dataset_id}'.`);
|
|
22361
23798
|
}
|
|
@@ -22435,7 +23872,8 @@ class PgPipelineRunStorage {
|
|
|
22435
23872
|
ownerColumn: "pipeline_id",
|
|
22436
23873
|
ownerIds: input.pipelineIds,
|
|
22437
23874
|
projectId: input.projectId,
|
|
22438
|
-
ownerIdFor: (row) => row.pipeline_id
|
|
23875
|
+
ownerIdFor: (row) => row.pipeline_id,
|
|
23876
|
+
excludedStatus: input.excludedStatus
|
|
22439
23877
|
});
|
|
22440
23878
|
return {
|
|
22441
23879
|
runs: rows.map(rowToPipelineRunRecord)
|
|
@@ -22456,6 +23894,11 @@ class PgPipelineRunStorage {
|
|
|
22456
23894
|
whereClauses.push(`pipeline_run_id = $${index++}`);
|
|
22457
23895
|
params.push(input.pipelineRunId);
|
|
22458
23896
|
}
|
|
23897
|
+
if (input.pipelineRunIds) {
|
|
23898
|
+
const placeholders = input.pipelineRunIds.map(() => `$${index++}`);
|
|
23899
|
+
whereClauses.push(`pipeline_run_id IN (${placeholders.join(", ") || "NULL"})`);
|
|
23900
|
+
params.push(...input.pipelineRunIds);
|
|
23901
|
+
}
|
|
22459
23902
|
if (input.pipelineId) {
|
|
22460
23903
|
whereClauses.push(`pipeline_id = $${index++}`);
|
|
22461
23904
|
params.push(input.pipelineId);
|
|
@@ -22487,6 +23930,9 @@ class PgPipelineRunStorage {
|
|
|
22487
23930
|
};
|
|
22488
23931
|
}
|
|
22489
23932
|
}
|
|
23933
|
+
function staleExecution(id) {
|
|
23934
|
+
return new PipelineRunError(`[SixbPg] Execution token is no longer current on pipeline run '${id}'.`);
|
|
23935
|
+
}
|
|
22490
23936
|
function rowToPipelineRunRecord(row) {
|
|
22491
23937
|
return {
|
|
22492
23938
|
id: row.id,
|
|
@@ -22494,6 +23940,7 @@ function rowToPipelineRunRecord(row) {
|
|
|
22494
23940
|
executionId: row.execution_id,
|
|
22495
23941
|
pipelineId: row.pipeline_id,
|
|
22496
23942
|
status: row.status,
|
|
23943
|
+
...pgRunExecutionFields(row),
|
|
22497
23944
|
queuedAt: new Date(row.queued_at),
|
|
22498
23945
|
startedAt: row.started_at ? new Date(row.started_at) : undefined,
|
|
22499
23946
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
@@ -22564,6 +24011,7 @@ import {
|
|
|
22564
24011
|
SUPERSEDING_PROJECTION_RUN_STATUSES,
|
|
22565
24012
|
staleProjectionRunExecution
|
|
22566
24013
|
} from "@sixb/core/internal/projection-run-storage-provider";
|
|
24014
|
+
import { laterLease as laterLease2 } from "@sixb/core/internal/storage";
|
|
22567
24015
|
import { PROJECTION_RUN_FAILURE_CODES, ProjectionRunError } from "@sixb/core/storage";
|
|
22568
24016
|
class PgProjectionRunStorage {
|
|
22569
24017
|
sql;
|
|
@@ -22659,11 +24107,12 @@ class PgProjectionRunStorage {
|
|
|
22659
24107
|
throw projectionRunNotFound(input.projectId, input.id);
|
|
22660
24108
|
const existing = restoreProjectionRunRow(existingRow);
|
|
22661
24109
|
const { attempt } = planProjectionRunReclaim(existing, input);
|
|
22662
|
-
const executionToken = createFreshExecutionToken(existing.executionToken);
|
|
24110
|
+
const executionToken = input.execution?.token ?? createFreshExecutionToken(existing.executionToken);
|
|
22663
24111
|
const [updated] = await tx`
|
|
22664
24112
|
UPDATE projection_runs
|
|
22665
24113
|
SET status = ${"running"}, started_at = COALESCE(started_at, ${input.startedAt ?? new Date}),
|
|
22666
|
-
finished_at = ${null}, attempt = ${attempt}, execution_token = ${executionToken}
|
|
24114
|
+
finished_at = ${null}, attempt = ${attempt}, execution_token = ${executionToken},
|
|
24115
|
+
execution_queue_lease_expires_at = ${input.execution?.queueLeaseExpiresAt ?? null}
|
|
22667
24116
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
22668
24117
|
AND (
|
|
22669
24118
|
(status = ${"queued"} AND execution_token IS NULL)
|
|
@@ -22676,6 +24125,45 @@ class PgProjectionRunStorage {
|
|
|
22676
24125
|
return projectionRunClaim(updated);
|
|
22677
24126
|
});
|
|
22678
24127
|
}
|
|
24128
|
+
async confirmExecutionOwnership(input) {
|
|
24129
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
24130
|
+
const [existingRow] = await tx`
|
|
24131
|
+
SELECT * FROM projection_runs
|
|
24132
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
24133
|
+
FOR UPDATE
|
|
24134
|
+
`;
|
|
24135
|
+
if (!existingRow)
|
|
24136
|
+
throw projectionRunNotFound(input.projectId, input.id);
|
|
24137
|
+
const existing = restoreProjectionRunRow(existingRow);
|
|
24138
|
+
if (existing.status !== "running" || existing.executionToken !== input.executionToken) {
|
|
24139
|
+
throw staleProjectionRunExecution(input.id);
|
|
24140
|
+
}
|
|
24141
|
+
const [updated] = await tx`
|
|
24142
|
+
UPDATE projection_runs
|
|
24143
|
+
SET execution_queue_lease_expires_at = ${laterLease2(existing.queueLeaseExpiresAt, input.queueLeaseExpiresAt)}
|
|
24144
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
24145
|
+
RETURNING *
|
|
24146
|
+
`;
|
|
24147
|
+
return rowToProjectionRunRecord(updated);
|
|
24148
|
+
});
|
|
24149
|
+
}
|
|
24150
|
+
async listLapsed(input) {
|
|
24151
|
+
const rows = await this.sql`
|
|
24152
|
+
SELECT * FROM projection_runs
|
|
24153
|
+
WHERE project_id = ${input.projectId} AND status = ${"running"}
|
|
24154
|
+
AND execution_queue_lease_expires_at <= ${input.lapsedBy}
|
|
24155
|
+
ORDER BY execution_queue_lease_expires_at
|
|
24156
|
+
LIMIT ${input.limit}
|
|
24157
|
+
`;
|
|
24158
|
+
return rows.map(rowToProjectionRunRecord);
|
|
24159
|
+
}
|
|
24160
|
+
async releaseExecution(input) {
|
|
24161
|
+
await this.sql`
|
|
24162
|
+
UPDATE projection_runs SET execution_queue_lease_expires_at = ${input.releasedAt ?? new Date}
|
|
24163
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"running"}
|
|
24164
|
+
AND execution_token = ${input.executionToken}
|
|
24165
|
+
`;
|
|
24166
|
+
}
|
|
22679
24167
|
async failEnqueue(input) {
|
|
22680
24168
|
return runPgTransaction(this.sql, async (tx) => {
|
|
22681
24169
|
const [existingRow] = await tx`
|
|
@@ -22759,6 +24247,7 @@ class PgProjectionRunStorage {
|
|
|
22759
24247
|
status = ${input.status},
|
|
22760
24248
|
finished_at = ${plan.finishedAt},
|
|
22761
24249
|
execution_token = ${null},
|
|
24250
|
+
execution_queue_lease_expires_at = ${null},
|
|
22762
24251
|
source_rows_read = ${plan.progress.sourceRowsRead},
|
|
22763
24252
|
source_rows_skipped = ${plan.progress.sourceRowsSkipped},
|
|
22764
24253
|
source_changes_read = ${plan.progress.sourceChangesRead ?? null},
|
|
@@ -23022,6 +24511,7 @@ function restoreProjectionRunRow(row) {
|
|
|
23022
24511
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
23023
24512
|
attempt: databaseSafeInteger2(row.attempt, "attempt"),
|
|
23024
24513
|
executionToken: row.execution_token ?? undefined,
|
|
24514
|
+
queueLeaseExpiresAt: row.execution_queue_lease_expires_at ? new Date(row.execution_queue_lease_expires_at) : undefined,
|
|
23025
24515
|
fixedBatchSize: optionalDatabaseSafeInteger(row.fixed_batch_size, "fixedBatchSize"),
|
|
23026
24516
|
nextBatchOrdinal: optionalDatabaseSafeInteger(row.next_batch_ordinal, "nextBatchOrdinal"),
|
|
23027
24517
|
nextRowOffset: optionalDatabaseSafeInteger(row.next_row_offset, "nextRowOffset"),
|
|
@@ -23296,9 +24786,21 @@ function toIsoString3(value) {
|
|
|
23296
24786
|
|
|
23297
24787
|
// src/pg-sync-run-storage.ts
|
|
23298
24788
|
import { parseSixbFailure as parseSixbFailure5, serializeSixbFailure as serializeSixbFailure5 } from "@sixb/core/internal/errors";
|
|
23299
|
-
import {
|
|
24789
|
+
import {
|
|
24790
|
+
canFinishSyncRun,
|
|
24791
|
+
canRequeueSyncRunAfterEnqueueFailure,
|
|
24792
|
+
mayFinishRun as mayFinishRun2
|
|
24793
|
+
} from "@sixb/core/internal/storage";
|
|
23300
24794
|
import { assertSyncRunExecution } from "@sixb/core/internal/sync-run-storage-provider";
|
|
23301
24795
|
import { SYNC_RUN_FAILURE_CODES, SyncRunError } from "@sixb/core/storage";
|
|
24796
|
+
var SYNC_RUNS = {
|
|
24797
|
+
name: "sync_runs",
|
|
24798
|
+
returning: "*, checkpoint IS NOT NULL AS checkpoint_present",
|
|
24799
|
+
toRun: rowToSyncRunRecord,
|
|
24800
|
+
notFound: (projectId, id) => new SyncRunError(`[SixbPg] Sync run '${id}' not found for project '${projectId}'.`),
|
|
24801
|
+
staleExecution: staleExecution2
|
|
24802
|
+
};
|
|
24803
|
+
|
|
23302
24804
|
class PgSyncRunStorage {
|
|
23303
24805
|
sql;
|
|
23304
24806
|
executions;
|
|
@@ -23364,7 +24866,9 @@ class PgSyncRunStorage {
|
|
|
23364
24866
|
const [updated] = await tx`
|
|
23365
24867
|
UPDATE sync_runs
|
|
23366
24868
|
SET status = ${"queued"}, queued_at = ${input.queuedAt ?? new Date},
|
|
23367
|
-
|
|
24869
|
+
attempt = ${0}, execution_token = ${null}, execution_queue_lease_expires_at = ${null},
|
|
24870
|
+
commit_prepared_at = ${null}, started_at = ${null}, finished_at = ${null},
|
|
24871
|
+
rows_read = ${null},
|
|
23368
24872
|
output_version_id = ${null}, error = ${null}, checkpoint = ${null}
|
|
23369
24873
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
23370
24874
|
RETURNING *, checkpoint IS NOT NULL AS checkpoint_present
|
|
@@ -23372,39 +24876,55 @@ class PgSyncRunStorage {
|
|
|
23372
24876
|
return rowToSyncRunRecord(updated);
|
|
23373
24877
|
});
|
|
23374
24878
|
}
|
|
23375
|
-
async
|
|
23376
|
-
const [updated] = await this.sql`
|
|
23377
|
-
UPDATE sync_runs
|
|
23378
|
-
SET status = ${"running"}, started_at = ${input.startedAt ?? new Date},
|
|
23379
|
-
error = ${null}
|
|
23380
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"queued"}
|
|
23381
|
-
RETURNING *, checkpoint IS NOT NULL AS checkpoint_present
|
|
23382
|
-
`;
|
|
23383
|
-
if (updated)
|
|
23384
|
-
return rowToSyncRunRecord(updated);
|
|
23385
|
-
const existing = await this.getById({ projectId: input.projectId, id: input.id });
|
|
23386
|
-
if (!existing) {
|
|
23387
|
-
throw new SyncRunError(`[SixbPg] Sync run '${input.id}' not found for project '${input.projectId}'.`);
|
|
23388
|
-
}
|
|
23389
|
-
throw new SyncRunError(`[SixbPg] Sync run '${input.id}' cannot start from status '${existing.status}'.`);
|
|
23390
|
-
}
|
|
23391
|
-
async finish(input) {
|
|
24879
|
+
async startOrReclaim(input) {
|
|
23392
24880
|
return runPgTransaction(this.sql, async (tx) => {
|
|
23393
|
-
const
|
|
24881
|
+
const { syncId } = await lockPgRun(tx, SYNC_RUNS, input.projectId, input.id);
|
|
24882
|
+
await lockAdvisoryKeys(tx, [syncRunsLockKey(input.projectId, syncId)]);
|
|
24883
|
+
const [sibling] = await tx`
|
|
23394
24884
|
SELECT *, checkpoint IS NOT NULL AS checkpoint_present FROM sync_runs
|
|
24885
|
+
WHERE project_id = ${input.projectId} AND sync_id = ${syncId}
|
|
24886
|
+
AND status = ${"running"} AND id <> ${input.id}
|
|
24887
|
+
LIMIT 1
|
|
24888
|
+
`;
|
|
24889
|
+
return claimPgRun(tx, SYNC_RUNS, input, sibling && rowToSyncRunRecord(sibling));
|
|
24890
|
+
});
|
|
24891
|
+
}
|
|
24892
|
+
async confirmExecutionOwnership(input) {
|
|
24893
|
+
return runPgTransaction(this.sql, (tx) => confirmPgRunExecution(tx, SYNC_RUNS, input));
|
|
24894
|
+
}
|
|
24895
|
+
async listLost(input) {
|
|
24896
|
+
return listLostPgRuns(this.sql, SYNC_RUNS, input);
|
|
24897
|
+
}
|
|
24898
|
+
async releaseExecution(input) {
|
|
24899
|
+
await releasePgRun(this.sql, SYNC_RUNS, input);
|
|
24900
|
+
}
|
|
24901
|
+
async prepareCommit(input) {
|
|
24902
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
24903
|
+
await lockHeldPgRun(tx, SYNC_RUNS, input);
|
|
24904
|
+
const [updated] = await tx`
|
|
24905
|
+
UPDATE sync_runs
|
|
24906
|
+
SET rows_read = ${input.rowsRead},
|
|
24907
|
+
checkpoint = ${serializeCheckpoint(input.checkpoint)}::text::jsonb,
|
|
24908
|
+
commit_prepared_at = ${input.preparedAt ?? new Date}
|
|
23395
24909
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
24910
|
+
RETURNING *, checkpoint IS NOT NULL AS checkpoint_present
|
|
23396
24911
|
`;
|
|
23397
|
-
|
|
23398
|
-
|
|
23399
|
-
|
|
23400
|
-
|
|
23401
|
-
|
|
24912
|
+
return rowToSyncRunRecord(updated);
|
|
24913
|
+
});
|
|
24914
|
+
}
|
|
24915
|
+
async finish(input) {
|
|
24916
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
24917
|
+
const record = await lockPgRun(tx, SYNC_RUNS, input.projectId, input.id);
|
|
24918
|
+
if (!canFinishSyncRun(record.status, input)) {
|
|
24919
|
+
throw new SyncRunError(`[SixbPg] Sync run '${input.id}' cannot finish from status '${record.status}'.`);
|
|
23402
24920
|
}
|
|
24921
|
+
if (!mayFinishRun2(record, input))
|
|
24922
|
+
throw staleExecution2(input.id);
|
|
23403
24923
|
if (input.status === "succeeded") {
|
|
23404
|
-
if (input.output && input.output.datasetId !==
|
|
23405
|
-
throw new SyncRunError(`[SixbPg] Sync run '${input.id}' output dataset '${input.output.datasetId}' does not match '${
|
|
24924
|
+
if (input.output && input.output.datasetId !== record.datasetId) {
|
|
24925
|
+
throw new SyncRunError(`[SixbPg] Sync run '${input.id}' output dataset '${input.output.datasetId}' does not match '${record.datasetId}'.`);
|
|
23406
24926
|
}
|
|
23407
|
-
if (!input.output && input.rowsRead !== 0 &&
|
|
24927
|
+
if (!input.output && input.rowsRead !== 0 && record.mode !== "merge") {
|
|
23408
24928
|
throw new SyncRunError(`[SixbPg] Sync run '${input.id}' may omit its output with rows read only for an initial merge no-op.`);
|
|
23409
24929
|
}
|
|
23410
24930
|
}
|
|
@@ -23412,6 +24932,8 @@ class PgSyncRunStorage {
|
|
|
23412
24932
|
UPDATE sync_runs
|
|
23413
24933
|
SET
|
|
23414
24934
|
status = ${input.status},
|
|
24935
|
+
execution_token = ${null},
|
|
24936
|
+
execution_queue_lease_expires_at = ${null},
|
|
23415
24937
|
finished_at = ${input.finishedAt ?? new Date},
|
|
23416
24938
|
rows_read = ${input.rowsRead},
|
|
23417
24939
|
output_version_id = ${input.output?.versionId ?? null},
|
|
@@ -23423,8 +24945,10 @@ class PgSyncRunStorage {
|
|
|
23423
24945
|
UPDATE sync_runs
|
|
23424
24946
|
SET
|
|
23425
24947
|
status = ${input.status},
|
|
24948
|
+
execution_token = ${null},
|
|
24949
|
+
execution_queue_lease_expires_at = ${null},
|
|
23426
24950
|
finished_at = ${input.finishedAt ?? new Date},
|
|
23427
|
-
rows_read = ${input.rowsRead ??
|
|
24951
|
+
rows_read = ${input.rowsRead ?? record.rowsRead ?? null},
|
|
23428
24952
|
output_version_id = ${null},
|
|
23429
24953
|
error = ${input.error === undefined ? null : serializeSixbFailure5(input.error, SYNC_RUN_FAILURE_CODES)}::text::jsonb,
|
|
23430
24954
|
checkpoint = ${null}
|
|
@@ -23498,6 +25022,12 @@ class PgSyncRunStorage {
|
|
|
23498
25022
|
};
|
|
23499
25023
|
}
|
|
23500
25024
|
}
|
|
25025
|
+
function staleExecution2(id) {
|
|
25026
|
+
return new SyncRunError(`[SixbPg] Execution token is no longer current on sync run '${id}'.`);
|
|
25027
|
+
}
|
|
25028
|
+
function syncRunsLockKey(projectId, syncId) {
|
|
25029
|
+
return `sync-runs:${JSON.stringify([projectId, syncId])}`;
|
|
25030
|
+
}
|
|
23501
25031
|
function serializeCheckpoint(checkpoint) {
|
|
23502
25032
|
return checkpoint !== undefined ? JSON.stringify(checkpoint) : null;
|
|
23503
25033
|
}
|
|
@@ -23510,6 +25040,8 @@ function rowToSyncRunRecord(row) {
|
|
|
23510
25040
|
datasetId: row.dataset_id,
|
|
23511
25041
|
mode: row.mode,
|
|
23512
25042
|
status: row.status,
|
|
25043
|
+
...pgRunExecutionFields(row),
|
|
25044
|
+
commitPreparedAt: row.commit_prepared_at ? new Date(row.commit_prepared_at) : undefined,
|
|
23513
25045
|
queuedAt: new Date(row.queued_at),
|
|
23514
25046
|
startedAt: row.started_at ? new Date(row.started_at) : undefined,
|
|
23515
25047
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
@@ -23667,10 +25199,12 @@ class PgWebhookRunStorage {
|
|
|
23667
25199
|
const [row] = await this.sql`
|
|
23668
25200
|
INSERT INTO webhook_runs (
|
|
23669
25201
|
project_id, id, execution_id, connector_id, webhook_id, status, method, route,
|
|
25202
|
+
execution_token, execution_lease_expires_at,
|
|
23670
25203
|
started_at, request_body_bytes, request_body_sha256, idempotency_key
|
|
23671
25204
|
) VALUES (
|
|
23672
25205
|
${input.projectId}, ${input.id}, ${input.executionId}, ${input.connectorId},
|
|
23673
25206
|
${input.webhookId}, ${"running"}, ${input.method}, ${input.route},
|
|
25207
|
+
${input.execution.token}, ${input.execution.leaseExpiresAt},
|
|
23674
25208
|
${input.startedAt ?? new Date}, ${input.requestBodyBytes},
|
|
23675
25209
|
${input.requestBodySha256}, ${input.idempotencyKey ?? null}
|
|
23676
25210
|
)
|
|
@@ -23694,22 +25228,31 @@ class PgWebhookRunStorage {
|
|
|
23694
25228
|
if (!existing) {
|
|
23695
25229
|
throw new WebhookRunError(`[SixbPg] Webhook run '${input.id}' not found for project '${input.projectId}'.`, "not_found");
|
|
23696
25230
|
}
|
|
23697
|
-
|
|
25231
|
+
const startedAt = input.startedAt ?? new Date;
|
|
25232
|
+
if (!canRetryWebhookRun(rowToWebhookRunRecord(existing), startedAt)) {
|
|
23698
25233
|
throw new WebhookRunError(`[SixbPg] Webhook run '${input.id}' cannot restart from status '${existing.status}'.`, "invalid_transition");
|
|
23699
25234
|
}
|
|
23700
25235
|
const [updated] = await tx`
|
|
23701
25236
|
UPDATE webhook_runs
|
|
23702
|
-
SET status = ${"running"},
|
|
23703
|
-
|
|
23704
|
-
|
|
25237
|
+
SET status = ${"running"}, execution_token = ${input.execution.token},
|
|
25238
|
+
execution_lease_expires_at = ${input.execution.leaseExpiresAt},
|
|
25239
|
+
started_at = ${startedAt}, finished_at = ${null}, response_status = ${null},
|
|
25240
|
+
error = ${null}
|
|
25241
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
23705
25242
|
RETURNING *
|
|
23706
25243
|
`;
|
|
23707
|
-
if (!updated) {
|
|
23708
|
-
throw new WebhookRunError(`[SixbPg] Webhook run '${input.id}' changed while restarting.`, "invalid_transition");
|
|
23709
|
-
}
|
|
23710
25244
|
return rowToWebhookRunRecord(updated);
|
|
23711
25245
|
});
|
|
23712
25246
|
}
|
|
25247
|
+
async renewLease(input) {
|
|
25248
|
+
const updated = await this.sql`
|
|
25249
|
+
UPDATE webhook_runs SET execution_lease_expires_at = ${input.leaseExpiresAt}
|
|
25250
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"running"}
|
|
25251
|
+
AND execution_token = ${input.executionToken}
|
|
25252
|
+
RETURNING *
|
|
25253
|
+
`;
|
|
25254
|
+
return updated.length > 0;
|
|
25255
|
+
}
|
|
23713
25256
|
async finish(input) {
|
|
23714
25257
|
return runPgTransaction(this.sql, async (tx) => {
|
|
23715
25258
|
const [existing] = await tx`
|
|
@@ -23723,9 +25266,13 @@ class PgWebhookRunStorage {
|
|
|
23723
25266
|
if (existing.status !== "running") {
|
|
23724
25267
|
throw new WebhookRunError(`[SixbPg] Webhook run '${input.id}' for project '${input.projectId}' is already terminal.`, "invalid_transition");
|
|
23725
25268
|
}
|
|
25269
|
+
if (existing.execution_token !== input.executionToken) {
|
|
25270
|
+
throw new WebhookRunError(`[SixbPg] Execution token is no longer current on webhook run '${input.id}'.`, "invalid_transition");
|
|
25271
|
+
}
|
|
23726
25272
|
const [updated] = await tx`
|
|
23727
25273
|
UPDATE webhook_runs
|
|
23728
|
-
SET status = ${input.status},
|
|
25274
|
+
SET status = ${input.status}, execution_token = ${null},
|
|
25275
|
+
execution_lease_expires_at = ${null}, finished_at = ${input.finishedAt ?? new Date},
|
|
23729
25276
|
response_status = ${input.responseStatus ?? null},
|
|
23730
25277
|
error = ${input.status === "succeeded" ? null : serializeSixbFailure6(input.error, WEBHOOK_RUN_FAILURE_CODES)}::text::jsonb
|
|
23731
25278
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
@@ -23791,6 +25338,10 @@ function rowToWebhookRunRecord(row) {
|
|
|
23791
25338
|
connectorId: row.connector_id,
|
|
23792
25339
|
webhookId: row.webhook_id,
|
|
23793
25340
|
status: row.status,
|
|
25341
|
+
execution: row.execution_token && row.execution_lease_expires_at ? {
|
|
25342
|
+
token: row.execution_token,
|
|
25343
|
+
leaseExpiresAt: new Date(row.execution_lease_expires_at)
|
|
25344
|
+
} : undefined,
|
|
23794
25345
|
method: row.method,
|
|
23795
25346
|
route: row.route,
|
|
23796
25347
|
startedAt: new Date(row.started_at),
|
|
@@ -24044,6 +25595,7 @@ function assertNonNegativeInteger(value, fieldName) {
|
|
|
24044
25595
|
|
|
24045
25596
|
// src/pg-workflow-run-storage.ts
|
|
24046
25597
|
import { parseSixbFailure as parseSixbFailure7, serializeSixbFailure as serializeSixbFailure7 } from "@sixb/core/internal/errors";
|
|
25598
|
+
import { holdsRun as holdsRun2, mayFinishRun as mayFinishRun3 } from "@sixb/core/internal/storage";
|
|
24047
25599
|
import {
|
|
24048
25600
|
assertWorkflowAgentNodeRunExecution,
|
|
24049
25601
|
assertWorkflowRunExecution
|
|
@@ -24053,6 +25605,14 @@ import {
|
|
|
24053
25605
|
WORKFLOW_RUN_FAILURE_CODES,
|
|
24054
25606
|
WorkflowRunError
|
|
24055
25607
|
} from "@sixb/core/storage";
|
|
25608
|
+
var WORKFLOW_RUNS = {
|
|
25609
|
+
name: "workflow_runs",
|
|
25610
|
+
returning: "*",
|
|
25611
|
+
toRun: rowToWorkflowRunRecord,
|
|
25612
|
+
notFound: (projectId, id) => new WorkflowRunError(`[SixbPg] Workflow run '${id}' not found for project '${projectId}'.`),
|
|
25613
|
+
staleExecution: (id) => new WorkflowRunError(`[SixbPg] Execution token is no longer current on workflow run '${id}'.`)
|
|
25614
|
+
};
|
|
25615
|
+
|
|
24056
25616
|
class PgWorkflowRunStorage {
|
|
24057
25617
|
sql;
|
|
24058
25618
|
executions;
|
|
@@ -24106,59 +25666,17 @@ class PgWorkflowRunStorage {
|
|
|
24106
25666
|
throw error;
|
|
24107
25667
|
}
|
|
24108
25668
|
}
|
|
24109
|
-
async
|
|
24110
|
-
return runPgTransaction(this.sql,
|
|
24111
|
-
const startedAt = input.startedAt ?? new Date;
|
|
24112
|
-
const [existing] = await tx`
|
|
24113
|
-
SELECT * FROM workflow_runs
|
|
24114
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
24115
|
-
FOR UPDATE
|
|
24116
|
-
`;
|
|
24117
|
-
if (!existing || existing.status !== "queued") {
|
|
24118
|
-
throw new WorkflowRunError(existing ? `[SixbPg] Workflow run '${input.id}' cannot start from status '${existing.status}'.` : `[SixbPg] Workflow run '${input.id}' not found for project '${input.projectId}'.`);
|
|
24119
|
-
}
|
|
24120
|
-
const [updated] = await tx`
|
|
24121
|
-
UPDATE workflow_runs
|
|
24122
|
-
SET
|
|
24123
|
-
status = ${"running"},
|
|
24124
|
-
started_at = ${startedAt},
|
|
24125
|
-
finished_at = ${null},
|
|
24126
|
-
error = ${null},
|
|
24127
|
-
attempt = attempt + 1,
|
|
24128
|
-
execution_token = ${input.execution?.token ?? null},
|
|
24129
|
-
execution_queue_lease_expires_at = ${input.execution?.queueLeaseExpiresAt ?? null}
|
|
24130
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
24131
|
-
RETURNING *
|
|
24132
|
-
`;
|
|
24133
|
-
return rowToWorkflowRunRecord(updated);
|
|
24134
|
-
});
|
|
25669
|
+
async startOrReclaim(input) {
|
|
25670
|
+
return runPgTransaction(this.sql, (tx) => claimPgRun(tx, WORKFLOW_RUNS, input));
|
|
24135
25671
|
}
|
|
24136
|
-
async
|
|
24137
|
-
return
|
|
24138
|
-
|
|
24139
|
-
|
|
24140
|
-
|
|
24141
|
-
attempt = ${Number(existing.attempt) + 1},
|
|
24142
|
-
execution_token = ${input.execution.token},
|
|
24143
|
-
execution_queue_lease_expires_at = ${input.execution.queueLeaseExpiresAt}
|
|
24144
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
24145
|
-
RETURNING *
|
|
24146
|
-
`;
|
|
24147
|
-
return rowToWorkflowRunRecord(updated);
|
|
24148
|
-
});
|
|
25672
|
+
async listLost(input) {
|
|
25673
|
+
return listLostPgRuns(this.sql, WORKFLOW_RUNS, input);
|
|
25674
|
+
}
|
|
25675
|
+
async releaseExecution(input) {
|
|
25676
|
+
await releasePgRun(this.sql, WORKFLOW_RUNS, input);
|
|
24149
25677
|
}
|
|
24150
25678
|
async confirmExecutionOwnership(input) {
|
|
24151
|
-
return runPgTransaction(this.sql,
|
|
24152
|
-
const existing = await lockWorkflowRun(tx, input.projectId, input.id, "running");
|
|
24153
|
-
assertWorkflowRunExecutionOwnership(existing, input.executionToken);
|
|
24154
|
-
const current = existing.execution_queue_lease_expires_at ? new Date(existing.execution_queue_lease_expires_at) : input.queueLeaseExpiresAt;
|
|
24155
|
-
const [updated] = await tx`
|
|
24156
|
-
UPDATE workflow_runs SET execution_queue_lease_expires_at = ${new Date(Math.max(current.getTime(), input.queueLeaseExpiresAt.getTime()))}
|
|
24157
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
24158
|
-
RETURNING *
|
|
24159
|
-
`;
|
|
24160
|
-
return rowToWorkflowRunRecord(updated);
|
|
24161
|
-
});
|
|
25679
|
+
return runPgTransaction(this.sql, (tx) => confirmPgRunExecution(tx, WORKFLOW_RUNS, input));
|
|
24162
25680
|
}
|
|
24163
25681
|
async finish(input) {
|
|
24164
25682
|
return runPgTransaction(this.sql, async (tx) => {
|
|
@@ -24173,7 +25691,9 @@ class PgWorkflowRunStorage {
|
|
|
24173
25691
|
if (!canFinishWorkflowRun(existing.status, input.status)) {
|
|
24174
25692
|
throw new WorkflowRunError(`[SixbPg] Workflow run '${input.id}' for project '${input.projectId}' cannot be finished from status '${existing.status}'.`);
|
|
24175
25693
|
}
|
|
24176
|
-
|
|
25694
|
+
if (!mayFinishRun3(rowToWorkflowRunRecord(existing), input)) {
|
|
25695
|
+
throw WORKFLOW_RUNS.staleExecution(input.id);
|
|
25696
|
+
}
|
|
24177
25697
|
const [updated] = input.status === "succeeded" ? await tx`
|
|
24178
25698
|
UPDATE workflow_runs
|
|
24179
25699
|
SET
|
|
@@ -24247,7 +25767,7 @@ class PgWorkflowRunStorage {
|
|
|
24247
25767
|
status = ${"running"},
|
|
24248
25768
|
finished_at = ${null},
|
|
24249
25769
|
error = ${null},
|
|
24250
|
-
attempt =
|
|
25770
|
+
attempt = ${1},
|
|
24251
25771
|
execution_token = ${input.execution?.token ?? null},
|
|
24252
25772
|
execution_queue_lease_expires_at = ${input.execution?.queueLeaseExpiresAt ?? null}
|
|
24253
25773
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
@@ -24703,13 +26223,7 @@ function rowToWorkflowRunRecord(row) {
|
|
|
24703
26223
|
startedAt: new Date(row.started_at),
|
|
24704
26224
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
24705
26225
|
error: row.error === null ? undefined : parseSixbFailure7(row.error, WORKFLOW_RUN_FAILURE_CODES),
|
|
24706
|
-
|
|
24707
|
-
...row.execution_token && row.execution_queue_lease_expires_at ? {
|
|
24708
|
-
execution: {
|
|
24709
|
-
token: row.execution_token,
|
|
24710
|
-
queueLeaseExpiresAt: new Date(row.execution_queue_lease_expires_at)
|
|
24711
|
-
}
|
|
24712
|
-
} : {}
|
|
26226
|
+
...pgRunExecutionFields(row)
|
|
24713
26227
|
};
|
|
24714
26228
|
}
|
|
24715
26229
|
function rowToWorkflowNodeRunRecord(row) {
|
|
@@ -24785,21 +26299,9 @@ function rowToWorkflowAgentNodeRunRecord(row) {
|
|
|
24785
26299
|
function parseJson2(value) {
|
|
24786
26300
|
return typeof value === "string" ? JSON.parse(value) : value;
|
|
24787
26301
|
}
|
|
24788
|
-
async function lockWorkflowRun(sql, projectId, id, status) {
|
|
24789
|
-
const [row] = await sql`
|
|
24790
|
-
SELECT * FROM workflow_runs WHERE project_id = ${projectId} AND id = ${id} FOR UPDATE
|
|
24791
|
-
`;
|
|
24792
|
-
if (!row)
|
|
24793
|
-
throw new WorkflowRunError(`[SixbPg] Workflow run '${id}' not found.`);
|
|
24794
|
-
if (row.status !== status) {
|
|
24795
|
-
throw new WorkflowRunError(`[SixbPg] Workflow run '${id}' must be ${status} (status '${row.status}').`);
|
|
24796
|
-
}
|
|
24797
|
-
return row;
|
|
24798
|
-
}
|
|
24799
26302
|
function assertWorkflowRunExecutionOwnership(row, token) {
|
|
24800
|
-
if (row
|
|
24801
|
-
throw
|
|
24802
|
-
}
|
|
26303
|
+
if (!holdsRun2(rowToWorkflowRunRecord(row), token))
|
|
26304
|
+
throw WORKFLOW_RUNS.staleExecution(row.id);
|
|
24803
26305
|
}
|
|
24804
26306
|
async function assertWorkflowNodeParentExecutionOwnership(sql, node, token) {
|
|
24805
26307
|
const [run] = await sql`
|
|
@@ -25012,6 +26514,7 @@ import {
|
|
|
25012
26514
|
normalizeGetShareSessionByIdInput,
|
|
25013
26515
|
normalizeRenewShareSessionIfValidInput,
|
|
25014
26516
|
normalizeRevokeShareSessionInput,
|
|
26517
|
+
normalizeSetShareSessionGuestNameInput,
|
|
25015
26518
|
normalizeShareSessionCreate,
|
|
25016
26519
|
parseShareSessionRecord
|
|
25017
26520
|
} from "@sixb/core/internal/share-session-storage-provider";
|
|
@@ -25071,6 +26574,21 @@ class PgShareSessionStorage {
|
|
|
25071
26574
|
`;
|
|
25072
26575
|
return row ? recordFromRow2(row) : null;
|
|
25073
26576
|
}
|
|
26577
|
+
async setGuestName(input) {
|
|
26578
|
+
const normalized = normalizeSetShareSessionGuestNameInput(input);
|
|
26579
|
+
const [row] = await this.sql`
|
|
26580
|
+
UPDATE share_sessions
|
|
26581
|
+
SET guest_name = ${normalized.name}
|
|
26582
|
+
WHERE project_id = ${normalized.projectId}
|
|
26583
|
+
AND id = ${normalized.id}
|
|
26584
|
+
AND grant_id = ${normalized.grantId}
|
|
26585
|
+
AND revoked_at IS NULL
|
|
26586
|
+
AND expires_at > ${normalized.now}
|
|
26587
|
+
AND absolute_expires_at > ${normalized.now}
|
|
26588
|
+
RETURNING *
|
|
26589
|
+
`;
|
|
26590
|
+
return row ? recordFromRow2(row) : null;
|
|
26591
|
+
}
|
|
25074
26592
|
async revoke(input) {
|
|
25075
26593
|
const normalized = normalizeRevokeShareSessionInput(input);
|
|
25076
26594
|
const current = await this.getById(normalized);
|
|
@@ -25099,7 +26617,8 @@ function recordFromRow2(row) {
|
|
|
25099
26617
|
createdAt: row.created_at,
|
|
25100
26618
|
expiresAt: row.expires_at,
|
|
25101
26619
|
absoluteExpiresAt: row.absolute_expires_at,
|
|
25102
|
-
...row.revoked_at === null ? {} : { revokedAt: row.revoked_at }
|
|
26620
|
+
...row.revoked_at === null ? {} : { revokedAt: row.revoked_at },
|
|
26621
|
+
...row.guest_name === null ? {} : { guestName: row.guest_name }
|
|
25103
26622
|
});
|
|
25104
26623
|
}
|
|
25105
26624
|
function invalidRecord2(message, cause) {
|
|
@@ -25136,8 +26655,11 @@ class PostgresStorage {
|
|
|
25136
26655
|
shareGrants;
|
|
25137
26656
|
shareSessions;
|
|
25138
26657
|
fileUploadSessions;
|
|
26658
|
+
datasetIngest;
|
|
25139
26659
|
fileDownloadGrants;
|
|
25140
26660
|
activity;
|
|
26661
|
+
maintenanceHeartbeats;
|
|
26662
|
+
lakeRetentionRuns;
|
|
25141
26663
|
migrators;
|
|
25142
26664
|
sql;
|
|
25143
26665
|
schemaName;
|
|
@@ -25198,8 +26720,11 @@ class PostgresStorage {
|
|
|
25198
26720
|
this.shareGrants = createOperationScopedFacade(stores.shareGrants, scope);
|
|
25199
26721
|
this.shareSessions = createOperationScopedFacade(stores.shareSessions, scope);
|
|
25200
26722
|
this.fileUploadSessions = createOperationScopedFacade(stores.fileUploadSessions, scope);
|
|
26723
|
+
this.datasetIngest = createOperationScopedFacade(stores.datasetIngest, scope);
|
|
25201
26724
|
this.fileDownloadGrants = createOperationScopedFacade(stores.fileDownloadGrants, scope);
|
|
25202
26725
|
this.activity = createOperationScopedFacade(stores.activity, scope);
|
|
26726
|
+
this.maintenanceHeartbeats = createOperationScopedFacade(stores.maintenanceHeartbeats, scope);
|
|
26727
|
+
this.lakeRetentionRuns = createOperationScopedFacade(stores.lakeRetentionRuns, scope);
|
|
25203
26728
|
registerPostgresStorageTestingAdapter(this, (durationMs) => stores.connectorConnections.advanceTimeForTesting(durationMs));
|
|
25204
26729
|
}
|
|
25205
26730
|
async ping() {
|
|
@@ -25318,8 +26843,11 @@ function createPostgresStores(sql, options) {
|
|
|
25318
26843
|
shareGrants,
|
|
25319
26844
|
shareSessions,
|
|
25320
26845
|
fileUploadSessions: new PgFileUploadSessionStorage(sql),
|
|
26846
|
+
datasetIngest: new PgDatasetIngestStorage(sql),
|
|
25321
26847
|
fileDownloadGrants: new PgFileDownloadGrantStorage(sql),
|
|
25322
|
-
activity: new PgActivityStorage(sql)
|
|
26848
|
+
activity: new PgActivityStorage(sql),
|
|
26849
|
+
maintenanceHeartbeats: new PgMaintenanceHeartbeatStorage(sql),
|
|
26850
|
+
lakeRetentionRuns: new PgLakeRetentionRunStorage(sql)
|
|
25323
26851
|
};
|
|
25324
26852
|
}
|
|
25325
26853
|
function resolveTimeoutMillis(value, label) {
|
|
@@ -25336,4 +26864,4 @@ export {
|
|
|
25336
26864
|
migratePostgresStorage
|
|
25337
26865
|
};
|
|
25338
26866
|
|
|
25339
|
-
//# debugId=
|
|
26867
|
+
//# debugId=3787CB9C13986BA564756E2164756E21
|