@sixb/pg 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/agents/computers.d.ts +8 -2
- package/dist/agents/computers.d.ts.map +1 -1
- package/dist/auth-storage/auth-storage.d.ts +2 -1
- package/dist/auth-storage/auth-storage.d.ts.map +1 -1
- package/dist/auth-storage/identities.d.ts +4 -0
- package/dist/auth-storage/identities.d.ts.map +1 -1
- package/dist/auth-storage/rows.d.ts +2 -1
- package/dist/auth-storage/rows.d.ts.map +1 -1
- package/dist/auth-storage/users.d.ts.map +1 -1
- package/dist/connector-connection-storage.d.ts.map +1 -1
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2377 -428
- package/dist/index.js.map +37 -30
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/conditional-reader.d.ts +7 -2
- package/dist/objects/conditional-reader.d.ts.map +1 -1
- package/dist/objects/partitions.d.ts.map +1 -1
- package/dist/objects/prepare-queries.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +14 -1
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/query-indexes.d.ts +13 -1
- package/dist/objects/query-indexes.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +14 -1
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/storage.d.ts +11 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/objects/visibility-indexes.d.ts +22 -0
- package/dist/objects/visibility-indexes.d.ts.map +1 -0
- package/dist/objects/visibility-statistics.d.ts +26 -0
- package/dist/objects/visibility-statistics.d.ts.map +1 -0
- package/dist/objects/visibility.d.ts +34 -0
- package/dist/objects/visibility.d.ts.map +1 -0
- package/dist/ontology-storage/materialization-session.d.ts +5 -0
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +14 -0
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/pg-artifact-storage.d.ts +14 -2
- package/dist/pg-artifact-storage.d.ts.map +1 -1
- package/dist/pg-client.d.ts.map +1 -1
- package/dist/pg-dataset-ingest-storage.d.ts +6 -0
- package/dist/pg-dataset-ingest-storage.d.ts.map +1 -0
- package/dist/pg-lake-retention-run-storage.d.ts +11 -0
- package/dist/pg-lake-retention-run-storage.d.ts.map +1 -0
- package/dist/pg-maintenance-heartbeat-storage.d.ts +11 -0
- package/dist/pg-maintenance-heartbeat-storage.d.ts.map +1 -0
- package/dist/pg-pipeline-run-storage.d.ts +5 -2
- package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts +5 -1
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/pg-sync-run-storage.d.ts +6 -2
- package/dist/pg-sync-run-storage.d.ts.map +1 -1
- package/dist/pg-webhook-run-storage.d.ts +2 -1
- package/dist/pg-webhook-run-storage.d.ts.map +1 -1
- package/dist/pg-workflow-run-storage.d.ts +4 -3
- package/dist/pg-workflow-run-storage.d.ts.map +1 -1
- package/dist/run-claim-sql.d.ts +37 -0
- package/dist/run-claim-sql.d.ts.map +1 -0
- package/dist/share-session-storage.d.ts +2 -1
- package/dist/share-session-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/computers.ts +54 -5
- package/src/auth-storage/auth-storage.ts +66 -0
- package/src/auth-storage/identities.ts +15 -0
- package/src/auth-storage/rows.ts +3 -0
- package/src/auth-storage/users.ts +3 -0
- package/src/connector-connection-storage.ts +33 -16
- package/src/index.ts +15 -0
- package/src/migrations/087-agent-computer-background-commands.sql +4 -0
- package/src/migrations/088-web-search-model-kind.sql +12 -0
- package/src/migrations/089-maintenance-heartbeats.sql +10 -0
- package/src/migrations/090-lake-retention-runs.sql +8 -0
- package/src/migrations/091-dataset-ingest-inbox.sql +38 -0
- package/src/migrations/092-run-recovery.sql +23 -0
- package/src/migrations/093-webhook-and-projection-leases.sql +10 -0
- package/src/migrations/094-artifact-board-previews.sql +14 -0
- package/src/migrations/095-artifact-open-versions.sql +101 -0
- package/src/migrations/096-artifact-documents.sql +11 -0
- package/src/migrations/097-artifact-frame-files.sql +8 -0
- package/src/migrations/098-artifact-comments.sql +36 -0
- package/src/migrations/099-object-artifact-refs.sql +12 -0
- package/src/migrations/100-shared-link-guests.sql +17 -0
- package/src/migrations/101-connector-connection-owners.sql +38 -0
- package/src/migrations/102-user-suspension-source.sql +12 -0
- package/src/migrations.ts +46 -0
- package/src/objects/conditional-reader.ts +26 -4
- package/src/objects/partitions.ts +17 -4
- package/src/objects/prepare-queries.ts +8 -3
- package/src/objects/query-compiler.ts +64 -23
- package/src/objects/query-indexes.ts +93 -27
- package/src/objects/read-scope.ts +147 -77
- package/src/objects/reader.ts +46 -18
- package/src/objects/storage.ts +53 -0
- package/src/objects/visibility-indexes.ts +222 -0
- package/src/objects/visibility-statistics.ts +73 -0
- package/src/objects/visibility.ts +135 -0
- package/src/ontology-storage/materialization-session.ts +4 -0
- package/src/ontology-storage/materialization-writer.ts +68 -1
- package/src/ontology-storage/materializations.ts +6 -2
- package/src/pg-artifact-storage.ts +798 -137
- package/src/pg-client.ts +4 -0
- package/src/pg-dataset-ingest-storage.ts +305 -0
- package/src/pg-lake-retention-run-storage.ts +40 -0
- package/src/pg-maintenance-heartbeat-storage.ts +53 -0
- package/src/pg-pipeline-run-storage.ts +85 -42
- package/src/pg-projection-run-storage.ts +61 -2
- package/src/pg-sync-run-storage.ts +100 -37
- package/src/pg-webhook-run-storage.ts +37 -11
- package/src/pg-workflow-run-storage.ts +39 -100
- package/src/run-claim-sql.ts +173 -0
- package/src/share-session-storage.ts +20 -0
package/dist/index.js
CHANGED
|
@@ -833,7 +833,11 @@ class PgAgentContextCheckpointStore {
|
|
|
833
833
|
}
|
|
834
834
|
|
|
835
835
|
// src/agents/computers.ts
|
|
836
|
-
import {
|
|
836
|
+
import {
|
|
837
|
+
AGENT_COMPUTER_BACKGROUND_COMMANDS_MAX,
|
|
838
|
+
assertAgentBackgroundCommandRef,
|
|
839
|
+
assertAgentComputerSession
|
|
840
|
+
} from "@sixb/core/internal/agent-run-storage-provider";
|
|
837
841
|
|
|
838
842
|
class PgAgentComputerSessionStore {
|
|
839
843
|
sql;
|
|
@@ -842,21 +846,53 @@ class PgAgentComputerSessionStore {
|
|
|
842
846
|
}
|
|
843
847
|
async hold(input) {
|
|
844
848
|
assertAgentComputerSession(input, "SixbPg");
|
|
849
|
+
const background = JSON.stringify(input.backgroundCommands ?? []);
|
|
845
850
|
await this.sql`
|
|
846
851
|
INSERT INTO agent_computer_sessions (
|
|
847
852
|
project_id, owner_principal_type, owner_principal_id, id,
|
|
848
|
-
computer_id, computer_name, folder, expires_at
|
|
853
|
+
computer_id, computer_name, folder, expires_at, background_commands
|
|
849
854
|
) VALUES (
|
|
850
855
|
${input.projectId}, ${input.ownerPrincipal.type}, ${input.ownerPrincipal.id}, ${input.id},
|
|
851
|
-
${input.computer.id}, ${input.computer.name}, ${input.computer.folder}, ${input.expiresAt}
|
|
856
|
+
${input.computer.id}, ${input.computer.name}, ${input.computer.folder}, ${input.expiresAt},
|
|
857
|
+
${background}::text::jsonb
|
|
852
858
|
)
|
|
853
859
|
ON CONFLICT (project_id, owner_principal_type, owner_principal_id, id)
|
|
854
860
|
DO UPDATE SET
|
|
855
861
|
computer_id = excluded.computer_id,
|
|
856
862
|
computer_name = excluded.computer_name,
|
|
857
863
|
folder = excluded.folder,
|
|
858
|
-
expires_at = excluded.expires_at
|
|
864
|
+
expires_at = excluded.expires_at,
|
|
865
|
+
background_commands = excluded.background_commands
|
|
866
|
+
`;
|
|
867
|
+
}
|
|
868
|
+
async requestStop(input) {
|
|
869
|
+
assertAgentBackgroundCommandRef(input.command, "SixbPg");
|
|
870
|
+
const { threadId, id, startedAt } = input.command;
|
|
871
|
+
const ref = JSON.stringify({ threadId, id, startedAt });
|
|
872
|
+
const [row] = await this.sql`
|
|
873
|
+
UPDATE agent_computer_sessions AS session
|
|
874
|
+
SET stop_requests = (
|
|
875
|
+
SELECT COALESCE(jsonb_agg(kept.value ORDER BY kept.position), '[]'::jsonb)
|
|
876
|
+
FROM (
|
|
877
|
+
SELECT request.value, request.position
|
|
878
|
+
FROM jsonb_array_elements(
|
|
879
|
+
CASE
|
|
880
|
+
WHEN session.stop_requests @> jsonb_build_array(${ref}::text::jsonb)
|
|
881
|
+
THEN session.stop_requests
|
|
882
|
+
ELSE session.stop_requests || jsonb_build_array(${ref}::text::jsonb)
|
|
883
|
+
END
|
|
884
|
+
) WITH ORDINALITY AS request(value, position)
|
|
885
|
+
ORDER BY request.position DESC
|
|
886
|
+
LIMIT ${AGENT_COMPUTER_BACKGROUND_COMMANDS_MAX}
|
|
887
|
+
) AS kept
|
|
888
|
+
)
|
|
889
|
+
WHERE project_id = ${input.projectId}
|
|
890
|
+
AND owner_principal_type = ${input.ownerPrincipal.type}
|
|
891
|
+
AND owner_principal_id = ${input.ownerPrincipal.id}
|
|
892
|
+
AND id = ${input.id}
|
|
893
|
+
RETURNING *
|
|
859
894
|
`;
|
|
895
|
+
return row ? rowToSession(row) : null;
|
|
860
896
|
}
|
|
861
897
|
async release(input) {
|
|
862
898
|
const [row] = await this.sql`
|
|
@@ -895,7 +931,9 @@ function rowToSession(row) {
|
|
|
895
931
|
id: row.id,
|
|
896
932
|
ownerPrincipal: { type: row.owner_principal_type, id: row.owner_principal_id },
|
|
897
933
|
computer: { id: row.computer_id, name: row.computer_name, folder: row.folder },
|
|
898
|
-
expiresAt: new Date(row.expires_at)
|
|
934
|
+
expiresAt: new Date(row.expires_at),
|
|
935
|
+
...row.background_commands.length > 0 ? { backgroundCommands: row.background_commands } : {},
|
|
936
|
+
...row.stop_requests.length > 0 ? { stopRequests: row.stop_requests } : {}
|
|
899
937
|
};
|
|
900
938
|
}
|
|
901
939
|
|
|
@@ -2386,6 +2424,7 @@ function rowToUserRecord(row) {
|
|
|
2386
2424
|
displayName: row.display_name ?? undefined,
|
|
2387
2425
|
avatarUrl: row.avatar_url ?? undefined,
|
|
2388
2426
|
status: row.status,
|
|
2427
|
+
...row.suspension_source === null ? {} : { suspensionSource: row.suspension_source },
|
|
2389
2428
|
createdAt: toDate(row.created_at),
|
|
2390
2429
|
updatedAt: toDate(row.updated_at)
|
|
2391
2430
|
};
|
|
@@ -3357,6 +3396,16 @@ class PgAuthUserIdentityStore {
|
|
|
3357
3396
|
`;
|
|
3358
3397
|
return rows.map(rowToIdentityRecord);
|
|
3359
3398
|
}
|
|
3399
|
+
async listForStrategy(params) {
|
|
3400
|
+
const rows = await this.sql`
|
|
3401
|
+
SELECT *
|
|
3402
|
+
FROM auth_user_identities
|
|
3403
|
+
WHERE project_id = ${params.projectId}
|
|
3404
|
+
AND strategy_id = ${params.strategyId}
|
|
3405
|
+
ORDER BY subject COLLATE "C" ASC
|
|
3406
|
+
`;
|
|
3407
|
+
return rows.map(rowToIdentityRecord);
|
|
3408
|
+
}
|
|
3360
3409
|
}
|
|
3361
3410
|
|
|
3362
3411
|
// src/auth-storage/invitations.ts
|
|
@@ -4079,6 +4128,7 @@ class PgAuthUserStore {
|
|
|
4079
4128
|
display_name,
|
|
4080
4129
|
avatar_url,
|
|
4081
4130
|
status,
|
|
4131
|
+
suspension_source,
|
|
4082
4132
|
created_at,
|
|
4083
4133
|
updated_at
|
|
4084
4134
|
) VALUES (
|
|
@@ -4088,6 +4138,7 @@ class PgAuthUserStore {
|
|
|
4088
4138
|
${input.displayName ?? null},
|
|
4089
4139
|
${input.avatarUrl ?? null},
|
|
4090
4140
|
${input.status ?? "active"},
|
|
4141
|
+
${input.status === "suspended" ? "manual" : null},
|
|
4091
4142
|
${createdAt},
|
|
4092
4143
|
${updatedAt}
|
|
4093
4144
|
)
|
|
@@ -4163,6 +4214,7 @@ class PgAuthUserStore {
|
|
|
4163
4214
|
const [row] = await tx`
|
|
4164
4215
|
UPDATE auth_users
|
|
4165
4216
|
SET status = ${input.status},
|
|
4217
|
+
suspension_source = ${input.status === "suspended" ? "manual" : null},
|
|
4166
4218
|
updated_at = ${updatedAt}
|
|
4167
4219
|
WHERE project_id = ${input.projectId}
|
|
4168
4220
|
AND id = ${input.id}
|
|
@@ -4459,6 +4511,7 @@ class PgAuthStorage {
|
|
|
4459
4511
|
display_name: user.displayName ?? null,
|
|
4460
4512
|
avatar_url: user.avatarUrl ?? null,
|
|
4461
4513
|
status: user.status,
|
|
4514
|
+
suspension_source: null,
|
|
4462
4515
|
created_at: user.createdAt,
|
|
4463
4516
|
updated_at: user.updatedAt
|
|
4464
4517
|
};
|
|
@@ -4527,6 +4580,7 @@ class PgAuthStorage {
|
|
|
4527
4580
|
const [updated] = await tx`
|
|
4528
4581
|
UPDATE auth_users
|
|
4529
4582
|
SET status = 'suspended',
|
|
4583
|
+
suspension_source = 'manual',
|
|
4530
4584
|
updated_at = ${input.suspendedAt}
|
|
4531
4585
|
WHERE project_id = ${input.projectId}
|
|
4532
4586
|
AND id = ${input.userId}
|
|
@@ -4540,6 +4594,58 @@ class PgAuthStorage {
|
|
|
4540
4594
|
return { user: rowToUserRecord(updated), previousUser: rowToUserRecord(existing) };
|
|
4541
4595
|
});
|
|
4542
4596
|
}
|
|
4597
|
+
async applyDirectoryVerdict(input) {
|
|
4598
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
4599
|
+
await lockAdvisoryKeys(tx, [authLockKey("sessions", input.projectId, input.userId)]);
|
|
4600
|
+
const existing = await getUserRowById(tx, { projectId: input.projectId, id: input.userId }, { forUpdate: true });
|
|
4601
|
+
if (!existing) {
|
|
4602
|
+
throw new AuthStorageError11("missing_user", `[Sixb] User '${input.userId}' not found for project '${input.projectId}'.`);
|
|
4603
|
+
}
|
|
4604
|
+
const previousUser = rowToUserRecord(existing);
|
|
4605
|
+
if (existing.status === "suspended" && existing.suspension_source !== "directory") {
|
|
4606
|
+
return { user: previousUser, previousUser };
|
|
4607
|
+
}
|
|
4608
|
+
if (input.verdict.status === "inactive") {
|
|
4609
|
+
if (existing.status === "suspended")
|
|
4610
|
+
return { user: previousUser, previousUser };
|
|
4611
|
+
const [suspended] = await tx`
|
|
4612
|
+
UPDATE auth_users
|
|
4613
|
+
SET status = 'suspended',
|
|
4614
|
+
suspension_source = 'directory',
|
|
4615
|
+
updated_at = ${input.appliedAt}
|
|
4616
|
+
WHERE project_id = ${input.projectId}
|
|
4617
|
+
AND id = ${input.userId}
|
|
4618
|
+
RETURNING *
|
|
4619
|
+
`;
|
|
4620
|
+
await revokeActiveSessionsForUser(tx, {
|
|
4621
|
+
projectId: input.projectId,
|
|
4622
|
+
userId: input.userId,
|
|
4623
|
+
revokedAt: input.appliedAt
|
|
4624
|
+
});
|
|
4625
|
+
return { user: rowToUserRecord(suspended), previousUser };
|
|
4626
|
+
}
|
|
4627
|
+
let row = existing;
|
|
4628
|
+
if (existing.status === "suspended") {
|
|
4629
|
+
[row] = await tx`
|
|
4630
|
+
UPDATE auth_users
|
|
4631
|
+
SET status = ${"active"},
|
|
4632
|
+
suspension_source = NULL,
|
|
4633
|
+
updated_at = ${input.appliedAt}
|
|
4634
|
+
WHERE project_id = ${input.projectId}
|
|
4635
|
+
AND id = ${input.userId}
|
|
4636
|
+
RETURNING *
|
|
4637
|
+
`;
|
|
4638
|
+
}
|
|
4639
|
+
await this.syncDirectoryGroups(tx, {
|
|
4640
|
+
completedAt: input.appliedAt,
|
|
4641
|
+
existing: [],
|
|
4642
|
+
groupIds: input.verdict.groupIds,
|
|
4643
|
+
projectId: input.projectId,
|
|
4644
|
+
userId: input.userId
|
|
4645
|
+
});
|
|
4646
|
+
return { user: rowToUserRecord(row), previousUser };
|
|
4647
|
+
});
|
|
4648
|
+
}
|
|
4543
4649
|
assertMagicLinkUsable(projectId, id, tokenHash, completedAt, row) {
|
|
4544
4650
|
if (!row) {
|
|
4545
4651
|
throw new AuthStorageError11("missing_magic_link", `[Sixb] Magic link '${id}' not found for project '${projectId}'.`);
|
|
@@ -4723,7 +4829,9 @@ function assertCompletableDeviceAuthorization(authorization, input) {
|
|
|
4723
4829
|
}
|
|
4724
4830
|
// src/connector-connection-storage.ts
|
|
4725
4831
|
import {
|
|
4726
|
-
|
|
4832
|
+
connectorConnectionOwnerFromColumns,
|
|
4833
|
+
DurableConnectorConnectionStorage,
|
|
4834
|
+
connectorConnectionOwnerColumns as ownerColumns
|
|
4727
4835
|
} from "@sixb/core/internal/connector-connection-storage-provider";
|
|
4728
4836
|
class PgConnectorConnectionStorage extends DurableConnectorConnectionStorage {
|
|
4729
4837
|
clock;
|
|
@@ -4779,15 +4887,17 @@ class PgConnectorConnectionPersistence {
|
|
|
4779
4887
|
return new Date(new Date(row.now).getTime() + this.clock.offsetMs);
|
|
4780
4888
|
}
|
|
4781
4889
|
async insertAuthorizationAttempt(record) {
|
|
4890
|
+
const owner = ownerColumns(record.owner);
|
|
4782
4891
|
try {
|
|
4783
4892
|
await this.sql`
|
|
4784
4893
|
INSERT INTO connector_authorization_attempts (
|
|
4785
|
-
project_id, connector_id, id, slot, initiated_by_execution_id,
|
|
4894
|
+
project_id, connector_id, id, owner_type, owner_id, slot, initiated_by_execution_id,
|
|
4786
4895
|
state_hash, code_verifier, redirect_uri, connection_run_id, return_to,
|
|
4787
4896
|
callback_binding_hash, reauthorization_id, reauthorization_revision,
|
|
4788
4897
|
reauthorization_connection_ids, created_at, expires_at
|
|
4789
4898
|
) VALUES (
|
|
4790
|
-
${record.projectId}, ${record.connectorId}, ${record.id},
|
|
4899
|
+
${record.projectId}, ${record.connectorId}, ${record.id},
|
|
4900
|
+
${owner.ownerType}, ${owner.ownerId}, ${record.slot},
|
|
4791
4901
|
${record.initiatedByExecutionId}, ${record.stateHash},
|
|
4792
4902
|
${record.codeVerifier === undefined ? null : jsonParameter(this.sql, record.codeVerifier)}, ${record.redirectUri},
|
|
4793
4903
|
${record.connectionRunId ?? null}, ${record.returnTo ?? null},
|
|
@@ -4849,13 +4959,14 @@ class PgConnectorConnectionPersistence {
|
|
|
4849
4959
|
const values = runValues(record);
|
|
4850
4960
|
await this.sql`
|
|
4851
4961
|
INSERT INTO connector_connection_runs (
|
|
4852
|
-
project_id, connector_id, id, kind, slot, initiated_by_execution_id,
|
|
4962
|
+
project_id, connector_id, id, kind, owner_type, owner_id, slot, initiated_by_execution_id,
|
|
4853
4963
|
status, waiting_for, processing_id, callback_started_at,
|
|
4854
4964
|
expires_at, authorization_id, cleanup_authorization_id, connections, error,
|
|
4855
4965
|
created_at, updated_at, finished_at
|
|
4856
4966
|
) VALUES (
|
|
4857
4967
|
${record.projectId}, ${record.connectorId}, ${record.id}, ${values.kind},
|
|
4858
|
-
${values.
|
|
4968
|
+
${values.ownerType}, ${values.ownerId}, ${values.slot},
|
|
4969
|
+
${values.initiatedByExecutionId}, ${values.status},
|
|
4859
4970
|
${values.waitingFor}, ${values.processingId},
|
|
4860
4971
|
${values.callbackStartedAt}, ${values.expiresAt}, ${values.authorizationId},
|
|
4861
4972
|
${values.cleanupAuthorizationId},
|
|
@@ -4889,8 +5000,8 @@ class PgConnectorConnectionPersistence {
|
|
|
4889
5000
|
const values = runValues(record);
|
|
4890
5001
|
const rows = await this.sql`
|
|
4891
5002
|
UPDATE connector_connection_runs
|
|
4892
|
-
SET kind = ${values.kind},
|
|
4893
|
-
initiated_by_execution_id = ${values.initiatedByExecutionId}, status = ${values.status},
|
|
5003
|
+
SET kind = ${values.kind}, owner_type = ${values.ownerType}, owner_id = ${values.ownerId},
|
|
5004
|
+
slot = ${values.slot}, initiated_by_execution_id = ${values.initiatedByExecutionId}, status = ${values.status},
|
|
4894
5005
|
waiting_for = ${values.waitingFor},
|
|
4895
5006
|
processing_id = ${values.processingId}, callback_started_at = ${values.callbackStartedAt},
|
|
4896
5007
|
expires_at = ${values.expiresAt}, authorization_id = ${values.authorizationId},
|
|
@@ -4967,14 +5078,15 @@ class PgConnectorConnectionPersistence {
|
|
|
4967
5078
|
assertChanged(rows, "connector authorization", record.id);
|
|
4968
5079
|
}
|
|
4969
5080
|
async insertConnection(record) {
|
|
5081
|
+
const owner = ownerColumns(record.owner);
|
|
4970
5082
|
try {
|
|
4971
5083
|
await this.sql`
|
|
4972
5084
|
INSERT INTO connector_connections (
|
|
4973
|
-
project_id, connector_id, id, authorization_id, slot, account,
|
|
5085
|
+
project_id, connector_id, id, authorization_id, owner_type, owner_id, slot, account,
|
|
4974
5086
|
account_id, status, disconnected_at, created_at, updated_at
|
|
4975
5087
|
) VALUES (
|
|
4976
5088
|
${record.projectId}, ${record.connectorId}, ${record.id}, ${record.authorizationId},
|
|
4977
|
-
${record.slot}, ${jsonParameter(this.sql, record.account)},
|
|
5089
|
+
${owner.ownerType}, ${owner.ownerId}, ${record.slot}, ${jsonParameter(this.sql, record.account)},
|
|
4978
5090
|
${record.account.id}, ${record.status}, ${record.disconnectedAt ?? null},
|
|
4979
5091
|
${record.createdAt}, ${record.updatedAt}
|
|
4980
5092
|
)
|
|
@@ -4993,9 +5105,11 @@ class PgConnectorConnectionPersistence {
|
|
|
4993
5105
|
async getConnectionBySelector(input) {
|
|
4994
5106
|
if (!this.scope.connectorId)
|
|
4995
5107
|
return null;
|
|
5108
|
+
const owner = ownerColumns(input.owner);
|
|
4996
5109
|
const rows = await this.sql`
|
|
4997
5110
|
SELECT * FROM connector_connections
|
|
4998
5111
|
WHERE project_id = ${this.scope.projectId} AND connector_id = ${this.scope.connectorId}
|
|
5112
|
+
AND owner_type = ${owner.ownerType} AND owner_id = ${owner.ownerId}
|
|
4999
5113
|
AND slot = ${input.slot}
|
|
5000
5114
|
${this.lockClause()}
|
|
5001
5115
|
`;
|
|
@@ -5031,9 +5145,11 @@ class PgConnectorConnectionPersistence {
|
|
|
5031
5145
|
return rows.map(connectionFromRow);
|
|
5032
5146
|
}
|
|
5033
5147
|
async updateConnection(record) {
|
|
5148
|
+
const owner = ownerColumns(record.owner);
|
|
5034
5149
|
const rows = await this.sql`
|
|
5035
5150
|
UPDATE connector_connections
|
|
5036
|
-
SET authorization_id = ${record.authorizationId},
|
|
5151
|
+
SET authorization_id = ${record.authorizationId}, owner_type = ${owner.ownerType},
|
|
5152
|
+
owner_id = ${owner.ownerId}, slot = ${record.slot},
|
|
5037
5153
|
account = ${jsonParameter(this.sql, record.account)},
|
|
5038
5154
|
account_id = ${record.account.id}, status = ${record.status},
|
|
5039
5155
|
disconnected_at = ${record.disconnectedAt ?? null}, updated_at = ${record.updatedAt}
|
|
@@ -5088,7 +5204,7 @@ function authorizationAttemptFromRow(row) {
|
|
|
5088
5204
|
id: row.id,
|
|
5089
5205
|
projectId: row.project_id,
|
|
5090
5206
|
connectorId: row.connector_id,
|
|
5091
|
-
owner:
|
|
5207
|
+
owner: connectorConnectionOwnerFromColumns(row.owner_type, row.owner_id),
|
|
5092
5208
|
slot: row.slot,
|
|
5093
5209
|
initiatedByExecutionId: row.initiated_by_execution_id,
|
|
5094
5210
|
stateHash: row.state_hash,
|
|
@@ -5107,6 +5223,7 @@ function authorizationAttemptFromRow(row) {
|
|
|
5107
5223
|
function runValues(record) {
|
|
5108
5224
|
return {
|
|
5109
5225
|
kind: record.kind,
|
|
5226
|
+
...ownerColumns(record.owner),
|
|
5110
5227
|
slot: record.slot,
|
|
5111
5228
|
initiatedByExecutionId: record.initiatedByExecutionId,
|
|
5112
5229
|
status: record.status,
|
|
@@ -5127,7 +5244,7 @@ function connectionRunFromRow(row) {
|
|
|
5127
5244
|
projectId: row.project_id,
|
|
5128
5245
|
connectorId: row.connector_id,
|
|
5129
5246
|
kind: row.kind,
|
|
5130
|
-
owner:
|
|
5247
|
+
owner: connectorConnectionOwnerFromColumns(row.owner_type, row.owner_id),
|
|
5131
5248
|
slot: row.slot,
|
|
5132
5249
|
initiatedByExecutionId: row.initiated_by_execution_id,
|
|
5133
5250
|
createdAt: date(row.created_at),
|
|
@@ -5257,7 +5374,7 @@ function connectionFromRow(row) {
|
|
|
5257
5374
|
projectId: row.project_id,
|
|
5258
5375
|
connectorId: row.connector_id,
|
|
5259
5376
|
authorizationId: row.authorization_id,
|
|
5260
|
-
owner:
|
|
5377
|
+
owner: connectorConnectionOwnerFromColumns(row.owner_type, row.owner_id),
|
|
5261
5378
|
slot: row.slot,
|
|
5262
5379
|
account: row.account,
|
|
5263
5380
|
status: row.status,
|
|
@@ -10168,6 +10285,408 @@ END $$;
|
|
|
10168
10285
|
ANALYZE objects;
|
|
10169
10286
|
`;
|
|
10170
10287
|
|
|
10288
|
+
// src/migrations/087-agent-computer-background-commands.sql
|
|
10289
|
+
var _087_agent_computer_background_commands_default = `-- What each open sixb reports of its threads' background commands, which outlive the turn that
|
|
10290
|
+
-- started them, and the ones someone asked to stop from the web: its renewals hand those back.
|
|
10291
|
+
ALTER TABLE agent_computer_sessions ADD COLUMN background_commands JSONB NOT NULL DEFAULT '[]';
|
|
10292
|
+
ALTER TABLE agent_computer_sessions ADD COLUMN stop_requests JSONB NOT NULL DEFAULT '[]';
|
|
10293
|
+
`;
|
|
10294
|
+
|
|
10295
|
+
// src/migrations/088-web-search-model-kind.sql
|
|
10296
|
+
var _088_web_search_model_kind_default = `ALTER TABLE ai_model_call_usage
|
|
10297
|
+
DROP CONSTRAINT ai_model_call_usage_model_kind_check;
|
|
10298
|
+
|
|
10299
|
+
ALTER TABLE ai_model_call_usage
|
|
10300
|
+
ADD CONSTRAINT ai_model_call_usage_model_kind_check
|
|
10301
|
+
CHECK (
|
|
10302
|
+
model_kind IS NULL
|
|
10303
|
+
OR model_kind IN (
|
|
10304
|
+
'language', 'image', 'video', 'embedding', 'decision', 'transcription', 'speech', 'reranking',
|
|
10305
|
+
'web-search'
|
|
10306
|
+
)
|
|
10307
|
+
);
|
|
10308
|
+
`;
|
|
10309
|
+
|
|
10310
|
+
// src/migrations/089-maintenance-heartbeats.sql
|
|
10311
|
+
var _089_maintenance_heartbeats_default = `-- One row per project and owner, rewritten by that owner's maintenance loop as it runs. Another
|
|
10312
|
+
-- process reads it to tell a stopped loop from a quiet one.
|
|
10313
|
+
CREATE TABLE maintenance_heartbeats (
|
|
10314
|
+
project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
|
|
10315
|
+
owner TEXT NOT NULL CHECK (owner IN ('orchestrator', 'embedded')),
|
|
10316
|
+
beat_at TIMESTAMPTZ NOT NULL,
|
|
10317
|
+
every_ms INTEGER NOT NULL CHECK (every_ms > 0),
|
|
10318
|
+
passes JSONB NOT NULL CHECK (jsonb_typeof(passes) = 'array'),
|
|
10319
|
+
PRIMARY KEY (project_id, owner)
|
|
10320
|
+
);
|
|
10321
|
+
`;
|
|
10322
|
+
|
|
10323
|
+
// src/migrations/090-lake-retention-runs.sql
|
|
10324
|
+
var _090_lake_retention_runs_default = `-- One row per project, rewritten by each automatic lake retention run. The API reads it to tell a
|
|
10325
|
+
-- request nobody consumes from one a worker ran.
|
|
10326
|
+
CREATE TABLE lake_retention_runs (
|
|
10327
|
+
project_id TEXT PRIMARY KEY CHECK (length(trim(project_id)) > 0),
|
|
10328
|
+
started_at TIMESTAMPTZ NOT NULL,
|
|
10329
|
+
finished_at TIMESTAMPTZ,
|
|
10330
|
+
failed BOOLEAN NOT NULL
|
|
10331
|
+
);
|
|
10332
|
+
`;
|
|
10333
|
+
|
|
10334
|
+
// src/migrations/091-dataset-ingest-inbox.sql
|
|
10335
|
+
var _091_dataset_ingest_inbox_default = `-- Changes \`sixb.datasets.ingest\` stored and no flush has written to the lake yet. A flush tags the
|
|
10336
|
+
-- deliveries it takes with its id, writes them as one dataset version, then deletes them. One
|
|
10337
|
+
-- refused when written stays, marked, until the orchestrator purges it.
|
|
10338
|
+
CREATE TABLE dataset_ingest_deliveries (
|
|
10339
|
+
id BIGSERIAL,
|
|
10340
|
+
project_id TEXT NOT NULL,
|
|
10341
|
+
dataset_id TEXT NOT NULL,
|
|
10342
|
+
execution_id TEXT NOT NULL,
|
|
10343
|
+
correlation_id TEXT NOT NULL,
|
|
10344
|
+
received_at TIMESTAMPTZ NOT NULL,
|
|
10345
|
+
change_count INTEGER NOT NULL CHECK (change_count > 0),
|
|
10346
|
+
definition_hash TEXT NOT NULL,
|
|
10347
|
+
-- Text, not JSONB: nothing reads inside it, and JSONB refuses strings JSON allows.
|
|
10348
|
+
changes TEXT NOT NULL,
|
|
10349
|
+
flush_id TEXT,
|
|
10350
|
+
rejected_at TIMESTAMPTZ,
|
|
10351
|
+
rejection TEXT,
|
|
10352
|
+
-- Led by the dataset, so no plan for one dataset walks the deliveries of another.
|
|
10353
|
+
PRIMARY KEY (project_id, dataset_id, id)
|
|
10354
|
+
);
|
|
10355
|
+
CREATE INDEX dataset_ingest_deliveries_by_flush
|
|
10356
|
+
ON dataset_ingest_deliveries (project_id, dataset_id, flush_id, id);
|
|
10357
|
+
|
|
10358
|
+
-- One row per dataset: its open flush request, the lease of the flush that holds it, the batch
|
|
10359
|
+
-- that flush took, and when a batch last completed. Locking this row orders requests against batches.
|
|
10360
|
+
CREATE TABLE dataset_ingest_state (
|
|
10361
|
+
project_id TEXT NOT NULL,
|
|
10362
|
+
dataset_id TEXT NOT NULL,
|
|
10363
|
+
request BIGINT NOT NULL DEFAULT 0,
|
|
10364
|
+
requested_at TIMESTAMPTZ,
|
|
10365
|
+
lease_owner TEXT,
|
|
10366
|
+
lease_expires_at TIMESTAMPTZ,
|
|
10367
|
+
flush_id TEXT,
|
|
10368
|
+
flush_started_at TIMESTAMPTZ,
|
|
10369
|
+
failure_streak INTEGER NOT NULL DEFAULT 0,
|
|
10370
|
+
last_flushed_at TIMESTAMPTZ,
|
|
10371
|
+
PRIMARY KEY (project_id, dataset_id)
|
|
10372
|
+
);
|
|
10373
|
+
`;
|
|
10374
|
+
|
|
10375
|
+
// src/migrations/092-run-recovery.sql
|
|
10376
|
+
var _092_run_recovery_default = `-- A sync or pipeline run now survives its worker: each execution holds the run under a token that
|
|
10377
|
+
-- fences its writes, and projects its queue lease here so another delivery can tell a live holder
|
|
10378
|
+
-- from a dead one. A run with neither was started by an older version: nothing backfills it, and the
|
|
10379
|
+
-- recovery pass settles it once its old worker cannot still be alive.
|
|
10380
|
+
--
|
|
10381
|
+
-- Every column is added without touching a row. Constraints are \`NOT VALID\`: rows that exist cannot
|
|
10382
|
+
-- break them, and validating would scan the table under this lock.
|
|
10383
|
+
ALTER TABLE sync_runs
|
|
10384
|
+
ADD COLUMN attempt BIGINT NOT NULL DEFAULT 0,
|
|
10385
|
+
ADD COLUMN execution_token TEXT,
|
|
10386
|
+
ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ,
|
|
10387
|
+
ADD COLUMN commit_prepared_at TIMESTAMPTZ,
|
|
10388
|
+
ADD CONSTRAINT sync_runs_attempt_check CHECK (attempt >= 0) NOT VALID,
|
|
10389
|
+
ADD CONSTRAINT sync_runs_execution_lease_check
|
|
10390
|
+
CHECK ((execution_token IS NULL) = (execution_queue_lease_expires_at IS NULL)) NOT VALID;
|
|
10391
|
+
|
|
10392
|
+
ALTER TABLE pipeline_runs
|
|
10393
|
+
ADD COLUMN attempt BIGINT NOT NULL DEFAULT 0,
|
|
10394
|
+
ADD COLUMN execution_token TEXT,
|
|
10395
|
+
ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ,
|
|
10396
|
+
ADD CONSTRAINT pipeline_runs_attempt_check CHECK (attempt >= 0) NOT VALID,
|
|
10397
|
+
ADD CONSTRAINT pipeline_runs_execution_lease_check
|
|
10398
|
+
CHECK ((execution_token IS NULL) = (execution_queue_lease_expires_at IS NULL)) NOT VALID;
|
|
10399
|
+
`;
|
|
10400
|
+
|
|
10401
|
+
// src/migrations/093-webhook-and-projection-leases.sql
|
|
10402
|
+
var _093_webhook_and_projection_leases_default = `-- A webhook handler runs in the request, with no queue behind it, so the process running it keeps
|
|
10403
|
+
-- its own lease. A run with none was started by an older API, and is judged from when it started.
|
|
10404
|
+
-- A projection run holds a lease as other runs do, so a duplicate delivery leaves a live run alone.
|
|
10405
|
+
--
|
|
10406
|
+
-- These add nullable columns, which touches no row.
|
|
10407
|
+
ALTER TABLE webhook_runs
|
|
10408
|
+
ADD COLUMN execution_token TEXT,
|
|
10409
|
+
ADD COLUMN execution_lease_expires_at TIMESTAMPTZ;
|
|
10410
|
+
|
|
10411
|
+
ALTER TABLE projection_runs ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ;
|
|
10412
|
+
`;
|
|
10413
|
+
|
|
10414
|
+
// src/migrations/094-artifact-board-previews.sql
|
|
10415
|
+
var _094_artifact_board_previews_default = `-- A picture of a canvas's board in one mode and size, kept under its render key: a hash of what the board
|
|
10416
|
+
-- shows, which its version's outline names. Versions that keep a board as it was share its
|
|
10417
|
+
-- picture, so no artifact owns one and deleting an artifact leaves them. The first picture kept
|
|
10418
|
+
-- under a key stays.
|
|
10419
|
+
CREATE TABLE artifact_board_previews (
|
|
10420
|
+
project_id TEXT NOT NULL,
|
|
10421
|
+
render_key TEXT NOT NULL CHECK (length(render_key) > 0),
|
|
10422
|
+
theme TEXT NOT NULL CHECK (theme IN ('light', 'dark')),
|
|
10423
|
+
-- \`full\`, a PNG of the board at its own size; \`small\`, a WebP at most 480 pixels wide.
|
|
10424
|
+
size TEXT NOT NULL CHECK (size IN ('full', 'small')),
|
|
10425
|
+
file JSONB NOT NULL,
|
|
10426
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
10427
|
+
PRIMARY KEY (project_id, render_key, theme, size)
|
|
10428
|
+
);
|
|
10429
|
+
`;
|
|
10430
|
+
|
|
10431
|
+
// src/migrations/095-artifact-open-versions.sql
|
|
10432
|
+
var _095_artifact_open_versions_default = `-- An artifact's latest version is open until someone creates a version, which freezes it: a write
|
|
10433
|
+
-- replaces an open version's content, and the first write after a freeze opens the next version.
|
|
10434
|
+
-- \`revision\` counts every change to an artifact and only grows; a version's is the artifact's
|
|
10435
|
+
-- when it last changed. Every existing version was published to stay as it is, so each is frozen
|
|
10436
|
+
-- when it was published, and an address pinned to one keeps naming what it named.
|
|
10437
|
+
ALTER TABLE artifacts ADD COLUMN revision INTEGER;
|
|
10438
|
+
UPDATE artifacts SET revision = latest_version;
|
|
10439
|
+
ALTER TABLE artifacts
|
|
10440
|
+
ALTER COLUMN revision SET NOT NULL,
|
|
10441
|
+
ADD CONSTRAINT artifacts_revision_check CHECK (revision > 0);
|
|
10442
|
+
|
|
10443
|
+
-- \`created_at\` is when a version opened, \`updated_at\` when its content was last written, and
|
|
10444
|
+
-- \`frozen_at\` when someone created it, \`frozen_by_*\`: an existing version by whoever published it.
|
|
10445
|
+
ALTER TABLE artifact_versions
|
|
10446
|
+
ADD COLUMN revision INTEGER,
|
|
10447
|
+
ADD COLUMN updated_at TIMESTAMPTZ,
|
|
10448
|
+
ADD COLUMN frozen_at TIMESTAMPTZ,
|
|
10449
|
+
ADD COLUMN frozen_by_principal_type TEXT
|
|
10450
|
+
CHECK (frozen_by_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
10451
|
+
ADD COLUMN frozen_by_principal_id TEXT CHECK (length(frozen_by_principal_id) > 0);
|
|
10452
|
+
UPDATE artifact_versions
|
|
10453
|
+
SET revision = version, updated_at = created_at, frozen_at = created_at,
|
|
10454
|
+
frozen_by_principal_type = author_principal_type, frozen_by_principal_id = author_principal_id;
|
|
10455
|
+
ALTER TABLE artifact_versions
|
|
10456
|
+
ALTER COLUMN revision SET NOT NULL,
|
|
10457
|
+
ALTER COLUMN updated_at SET NOT NULL,
|
|
10458
|
+
ADD CONSTRAINT artifact_versions_revision_check CHECK (revision > 0),
|
|
10459
|
+
ADD CONSTRAINT artifact_versions_frozen_by_check CHECK (
|
|
10460
|
+
(frozen_at IS NULL) = (frozen_by_principal_type IS NULL)
|
|
10461
|
+
AND (frozen_by_principal_type IS NULL) = (frozen_by_principal_id IS NULL)
|
|
10462
|
+
);
|
|
10463
|
+
|
|
10464
|
+
-- At most one open version: the latest, which storage keeps so.
|
|
10465
|
+
CREATE UNIQUE INDEX idx_artifact_versions_open
|
|
10466
|
+
ON artifact_versions (project_id, artifact_id) WHERE frozen_at IS NULL;
|
|
10467
|
+
|
|
10468
|
+
-- A retry no longer finds what a turn wrote by its versions' origins, which the next write
|
|
10469
|
+
-- replaces: \`artifact_turn_writes\` keeps it.
|
|
10470
|
+
DROP INDEX idx_artifact_versions_origin_run;
|
|
10471
|
+
DROP INDEX idx_artifact_versions_origin_thread;
|
|
10472
|
+
|
|
10473
|
+
-- A frame link serves one revision of its version, and of each file it lends. The links issued
|
|
10474
|
+
-- before revisions name none, and expire within the hour: they go now, and the app asks again.
|
|
10475
|
+
DELETE FROM artifact_frame_grants;
|
|
10476
|
+
ALTER TABLE artifact_frame_grants
|
|
10477
|
+
ADD COLUMN revision INTEGER NOT NULL CHECK (revision > 0);
|
|
10478
|
+
|
|
10479
|
+
-- The Agent turn whose write shared an artifact in a thread: a retry of the turn takes the link
|
|
10480
|
+
-- back. Links made before turns were kept stay for as long as the artifact lasts.
|
|
10481
|
+
ALTER TABLE artifact_threads ADD COLUMN linked_by_run_id TEXT;
|
|
10482
|
+
CREATE INDEX idx_artifact_threads_linked_by_run
|
|
10483
|
+
ON artifact_threads (project_id, linked_by_run_id) WHERE linked_by_run_id IS NOT NULL;
|
|
10484
|
+
|
|
10485
|
+
-- What an artifact's latest version held before a change that can be undone: an Agent turn's
|
|
10486
|
+
-- first write to it, one per turn, or a restore. Kept by the revision whose content it holds; each
|
|
10487
|
+
-- artifact keeps its newest 20. \`refs\`, as in \`artifact_versions\`.
|
|
10488
|
+
CREATE TABLE artifact_restore_points (
|
|
10489
|
+
project_id TEXT NOT NULL,
|
|
10490
|
+
artifact_id TEXT NOT NULL,
|
|
10491
|
+
revision INTEGER NOT NULL CHECK (revision > 0),
|
|
10492
|
+
version INTEGER NOT NULL CHECK (version > 0),
|
|
10493
|
+
frozen BOOLEAN NOT NULL,
|
|
10494
|
+
title TEXT NOT NULL CHECK (length(trim(title)) > 0),
|
|
10495
|
+
entry TEXT NOT NULL CHECK (length(entry) > 0),
|
|
10496
|
+
files JSONB NOT NULL CHECK (jsonb_typeof(files) = 'array'),
|
|
10497
|
+
outline JSONB NOT NULL CHECK (jsonb_typeof(outline) = 'object'),
|
|
10498
|
+
refs JSONB NOT NULL CHECK (jsonb_typeof(refs) = 'array'),
|
|
10499
|
+
author_principal_type TEXT NOT NULL
|
|
10500
|
+
CHECK (author_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
10501
|
+
author_principal_id TEXT NOT NULL CHECK (length(author_principal_id) > 0),
|
|
10502
|
+
origin_thread_id TEXT,
|
|
10503
|
+
origin_run_id TEXT,
|
|
10504
|
+
updated_at TIMESTAMPTZ NOT NULL,
|
|
10505
|
+
reason TEXT NOT NULL CHECK (reason IN ('turn', 'restore')),
|
|
10506
|
+
turn_run_id TEXT,
|
|
10507
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
10508
|
+
PRIMARY KEY (project_id, artifact_id, revision),
|
|
10509
|
+
FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE,
|
|
10510
|
+
CHECK ((origin_thread_id IS NULL) = (origin_run_id IS NULL)),
|
|
10511
|
+
CHECK ((reason = 'turn') = (turn_run_id IS NOT NULL))
|
|
10512
|
+
);
|
|
10513
|
+
|
|
10514
|
+
CREATE UNIQUE INDEX idx_artifact_restore_points_turn
|
|
10515
|
+
ON artifact_restore_points (project_id, artifact_id, turn_run_id) WHERE turn_run_id IS NOT NULL;
|
|
10516
|
+
|
|
10517
|
+
-- The artifacts each Agent turn wrote, its sub-agents' writes among them, the revision its last
|
|
10518
|
+
-- write left and how many changes it made. Every change moves an artifact's revision by one, so a
|
|
10519
|
+
-- retry undoes an artifact only while those changes are all its revision moved by since the turn
|
|
10520
|
+
-- began: anyone's change in between keeps it. Each artifact keeps its newest 20 turns.
|
|
10521
|
+
CREATE TABLE artifact_turn_writes (
|
|
10522
|
+
project_id TEXT NOT NULL,
|
|
10523
|
+
artifact_id TEXT NOT NULL,
|
|
10524
|
+
turn_run_id TEXT NOT NULL CHECK (length(turn_run_id) > 0),
|
|
10525
|
+
created BOOLEAN NOT NULL,
|
|
10526
|
+
revision INTEGER NOT NULL CHECK (revision > 0),
|
|
10527
|
+
writes INTEGER NOT NULL CHECK (writes > 0),
|
|
10528
|
+
PRIMARY KEY (project_id, artifact_id, turn_run_id),
|
|
10529
|
+
FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE
|
|
10530
|
+
);
|
|
10531
|
+
|
|
10532
|
+
CREATE INDEX idx_artifact_turn_writes_turn ON artifact_turn_writes (project_id, turn_run_id);
|
|
10533
|
+
`;
|
|
10534
|
+
|
|
10535
|
+
// src/migrations/096-artifact-documents.sql
|
|
10536
|
+
var _096_artifact_documents_default = `-- A canvas's content as the Yjs document everyone editing it shares: its whole encoded state, in a
|
|
10537
|
+
-- blob, replaced by each write that changes the content, and the revision whose content it holds.
|
|
10538
|
+
-- It is not a journal of changes, only the document's own encoding, so a write merges into the
|
|
10539
|
+
-- same history wherever it is made. One whose revision is not the artifact's was left behind by a
|
|
10540
|
+
-- write that stored none: it is brought up to the content before it is used. Null until a write
|
|
10541
|
+
-- stores one, and for kinds that are not edited live.
|
|
10542
|
+
ALTER TABLE artifacts
|
|
10543
|
+
ADD COLUMN document JSONB CHECK (document IS NULL OR jsonb_typeof(document) = 'object'),
|
|
10544
|
+
ADD COLUMN document_revision INTEGER CHECK (document_revision > 0),
|
|
10545
|
+
ADD CONSTRAINT artifacts_document_pair_check
|
|
10546
|
+
CHECK ((document IS NULL) = (document_revision IS NULL));
|
|
10547
|
+
`;
|
|
10548
|
+
|
|
10549
|
+
// src/migrations/097-artifact-frame-files.sql
|
|
10550
|
+
var _097_artifact_frame_files_default = `-- A frame link names the files it serves, as a SHA-256 hex of their paths and digests, in place of
|
|
10551
|
+
-- the revision it was issued at: a canvas edited live is written every few seconds, and a link
|
|
10552
|
+
-- stays good while its boards' files are the same. Links last an hour, so those issued before are
|
|
10553
|
+
-- dropped.
|
|
10554
|
+
DELETE FROM artifact_frame_grants;
|
|
10555
|
+
ALTER TABLE artifact_frame_grants
|
|
10556
|
+
DROP COLUMN revision,
|
|
10557
|
+
ADD COLUMN files_digest TEXT NOT NULL CHECK (length(files_digest) = 64);
|
|
10558
|
+
`;
|
|
10559
|
+
|
|
10560
|
+
// src/migrations/098-artifact-comments.sql
|
|
10561
|
+
var _098_artifact_comments_default = `-- Comments on an artifact, and replies to them (\`parent_id\`), each about a board or an element of
|
|
10562
|
+
-- a canvas, or the artifact itself, at the revision its author saw (\`anchor\`). They go with the
|
|
10563
|
+
-- artifact, and a reply with what it replies to; a copy of the artifact keeps none.
|
|
10564
|
+
-- \`seq\` orders comments made in the same instant as they were made.
|
|
10565
|
+
CREATE TABLE artifact_comments (
|
|
10566
|
+
seq BIGINT GENERATED ALWAYS AS IDENTITY,
|
|
10567
|
+
project_id TEXT NOT NULL,
|
|
10568
|
+
artifact_id TEXT NOT NULL,
|
|
10569
|
+
id TEXT NOT NULL CHECK (length(id) > 0),
|
|
10570
|
+
parent_id TEXT,
|
|
10571
|
+
anchor JSONB NOT NULL CHECK (jsonb_typeof(anchor) = 'object'),
|
|
10572
|
+
body TEXT NOT NULL CHECK (length(body) > 0),
|
|
10573
|
+
author_principal_type TEXT NOT NULL
|
|
10574
|
+
CHECK (author_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
10575
|
+
author_principal_id TEXT NOT NULL CHECK (length(author_principal_id) > 0),
|
|
10576
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
10577
|
+
resolved_at TIMESTAMPTZ,
|
|
10578
|
+
resolved_by_principal_type TEXT
|
|
10579
|
+
CHECK (resolved_by_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
10580
|
+
resolved_by_principal_id TEXT CHECK (length(resolved_by_principal_id) > 0),
|
|
10581
|
+
-- The Agent's run that wrote it for its author, and that run's conversation.
|
|
10582
|
+
origin_thread_id TEXT,
|
|
10583
|
+
origin_run_id TEXT,
|
|
10584
|
+
PRIMARY KEY (project_id, artifact_id, id),
|
|
10585
|
+
FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE,
|
|
10586
|
+
FOREIGN KEY (project_id, artifact_id, parent_id)
|
|
10587
|
+
REFERENCES artifact_comments (project_id, artifact_id, id) ON DELETE CASCADE,
|
|
10588
|
+
CHECK (
|
|
10589
|
+
(resolved_at IS NULL) = (resolved_by_principal_type IS NULL)
|
|
10590
|
+
AND (resolved_by_principal_type IS NULL) = (resolved_by_principal_id IS NULL)
|
|
10591
|
+
),
|
|
10592
|
+
CHECK ((origin_thread_id IS NULL) = (origin_run_id IS NULL))
|
|
10593
|
+
);
|
|
10594
|
+
|
|
10595
|
+
CREATE INDEX idx_artifact_comments_created
|
|
10596
|
+
ON artifact_comments (project_id, artifact_id, created_at, seq);
|
|
10597
|
+
`;
|
|
10598
|
+
|
|
10599
|
+
// src/migrations/099-object-artifact-refs.sql
|
|
10600
|
+
var _099_object_artifact_refs_default = `-- The artifacts objects hold: one row per artifact an object property holds, wherever it sits in
|
|
10601
|
+
-- the property's value. Every object write keeps them current in its own transaction, so whoever
|
|
10602
|
+
-- reaches an artifact through the objects that hold it finds them here. Holders list by code
|
|
10603
|
+
-- point, which the "C" collation keeps in the key's order.
|
|
10604
|
+
CREATE TABLE object_artifact_refs (
|
|
10605
|
+
project_id TEXT NOT NULL,
|
|
10606
|
+
artifact_id TEXT NOT NULL,
|
|
10607
|
+
object_type_id TEXT COLLATE "C" NOT NULL,
|
|
10608
|
+
primary_id TEXT COLLATE "C" NOT NULL,
|
|
10609
|
+
property_id TEXT COLLATE "C" NOT NULL,
|
|
10610
|
+
PRIMARY KEY (project_id, artifact_id, object_type_id, primary_id, property_id)
|
|
10611
|
+
);
|
|
10612
|
+
`;
|
|
10613
|
+
|
|
10614
|
+
// src/migrations/100-shared-link-guests.sql
|
|
10615
|
+
var _100_shared_link_guests_default = `-- A shared link's visitor has no account: they give a name once, which their session keeps.
|
|
10616
|
+
ALTER TABLE share_sessions
|
|
10617
|
+
ADD COLUMN guest_name TEXT CHECK (guest_name IS NULL OR length(guest_name) > 0);
|
|
10618
|
+
|
|
10619
|
+
-- They comment as a guest: the author's id is their session, and the comment keeps the name they
|
|
10620
|
+
-- gave, whatever becomes of the session.
|
|
10621
|
+
ALTER TABLE artifact_comments
|
|
10622
|
+
DROP CONSTRAINT artifact_comments_author_principal_type_check;
|
|
10623
|
+
|
|
10624
|
+
ALTER TABLE artifact_comments
|
|
10625
|
+
ADD COLUMN author_name TEXT,
|
|
10626
|
+
ADD CONSTRAINT artifact_comments_author_principal_type_check
|
|
10627
|
+
CHECK (author_principal_type IN ('user', 'serviceAccount', 'system', 'guest')),
|
|
10628
|
+
ADD CONSTRAINT artifact_comments_author_name_check CHECK (
|
|
10629
|
+
(author_principal_type = 'guest') = (author_name IS NOT NULL)
|
|
10630
|
+
AND (author_name IS NULL OR length(author_name) > 0)
|
|
10631
|
+
);
|
|
10632
|
+
`;
|
|
10633
|
+
|
|
10634
|
+
// src/migrations/101-connector-connection-owners.sql
|
|
10635
|
+
var _101_connector_connection_owners_default = `-- A connection belongs to the project or to one person. The project's owner_id is empty, so the
|
|
10636
|
+
-- unique key over (owner_type, owner_id, slot) never meets a NULL. Existing rows become the
|
|
10637
|
+
-- project's; the defaults are dropped afterwards so every write has to name its owner.
|
|
10638
|
+
ALTER TABLE connector_connections
|
|
10639
|
+
ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
|
|
10640
|
+
ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
|
|
10641
|
+
|
|
10642
|
+
ALTER TABLE connector_connections
|
|
10643
|
+
ALTER COLUMN owner_type DROP DEFAULT,
|
|
10644
|
+
ALTER COLUMN owner_id DROP DEFAULT,
|
|
10645
|
+
ADD CONSTRAINT connector_connections_owner_check CHECK (
|
|
10646
|
+
(owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
|
|
10647
|
+
),
|
|
10648
|
+
DROP CONSTRAINT connector_connections_project_id_connector_id_slot_key,
|
|
10649
|
+
ADD CONSTRAINT connector_connections_owner_slot_key
|
|
10650
|
+
UNIQUE (project_id, connector_id, owner_type, owner_id, slot);
|
|
10651
|
+
|
|
10652
|
+
ALTER TABLE connector_connection_runs
|
|
10653
|
+
ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
|
|
10654
|
+
ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
|
|
10655
|
+
|
|
10656
|
+
ALTER TABLE connector_connection_runs
|
|
10657
|
+
ALTER COLUMN owner_type DROP DEFAULT,
|
|
10658
|
+
ALTER COLUMN owner_id DROP DEFAULT,
|
|
10659
|
+
ADD CONSTRAINT connector_connection_runs_owner_check CHECK (
|
|
10660
|
+
(owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
|
|
10661
|
+
);
|
|
10662
|
+
|
|
10663
|
+
ALTER TABLE connector_authorization_attempts
|
|
10664
|
+
ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
|
|
10665
|
+
ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
|
|
10666
|
+
|
|
10667
|
+
ALTER TABLE connector_authorization_attempts
|
|
10668
|
+
ALTER COLUMN owner_type DROP DEFAULT,
|
|
10669
|
+
ALTER COLUMN owner_id DROP DEFAULT,
|
|
10670
|
+
ADD CONSTRAINT connector_authorization_attempts_owner_check CHECK (
|
|
10671
|
+
(owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
|
|
10672
|
+
);
|
|
10673
|
+
`;
|
|
10674
|
+
|
|
10675
|
+
// src/migrations/102-user-suspension-source.sql
|
|
10676
|
+
var _102_user_suspension_source_default = `-- Who suspended a user: an administrator or server code ('manual'), or the identity provider's
|
|
10677
|
+
-- directory ('directory'), which may lift only its own suspensions. Only a suspended user has one,
|
|
10678
|
+
-- and every suspension made before this column existed was manual.
|
|
10679
|
+
ALTER TABLE auth_users ADD COLUMN suspension_source TEXT;
|
|
10680
|
+
|
|
10681
|
+
UPDATE auth_users SET suspension_source = 'manual' WHERE status = 'suspended';
|
|
10682
|
+
|
|
10683
|
+
ALTER TABLE auth_users
|
|
10684
|
+
ADD CONSTRAINT auth_users_suspension_source_check CHECK (
|
|
10685
|
+
(status = 'suspended' AND suspension_source IN ('manual', 'directory'))
|
|
10686
|
+
OR (status = 'active' AND suspension_source IS NULL)
|
|
10687
|
+
);
|
|
10688
|
+
`;
|
|
10689
|
+
|
|
10171
10690
|
// src/migrations.ts
|
|
10172
10691
|
var POSTGRES_STORAGE_ADAPTER_ID = "SixbPostgresStorage";
|
|
10173
10692
|
function pgSql(id, sqlText) {
|
|
@@ -10420,7 +10939,23 @@ var postgresStorageMigrations = defineMigrations({
|
|
|
10420
10939
|
pgSql("083-agent-thread-runs-on", _083_agent_thread_runs_on_default),
|
|
10421
10940
|
pgSql("084-agent-token-rates", _084_agent_token_rates_default),
|
|
10422
10941
|
pgSql("085-agent-computer-sessions", _085_agent_computer_sessions_default),
|
|
10423
|
-
pgSql("086-partition-objects", _086_partition_objects_default)
|
|
10942
|
+
pgSql("086-partition-objects", _086_partition_objects_default),
|
|
10943
|
+
pgSql("087-agent-computer-background-commands", _087_agent_computer_background_commands_default),
|
|
10944
|
+
pgSql("088-web-search-model-kind", _088_web_search_model_kind_default),
|
|
10945
|
+
pgSql("089-maintenance-heartbeats", _089_maintenance_heartbeats_default),
|
|
10946
|
+
pgSql("090-lake-retention-runs", _090_lake_retention_runs_default),
|
|
10947
|
+
pgSql("091-dataset-ingest-inbox", _091_dataset_ingest_inbox_default),
|
|
10948
|
+
pgSql("092-run-recovery", _092_run_recovery_default),
|
|
10949
|
+
pgSql("093-webhook-and-projection-leases", _093_webhook_and_projection_leases_default),
|
|
10950
|
+
pgSql("094-artifact-board-previews", _094_artifact_board_previews_default),
|
|
10951
|
+
pgSql("095-artifact-open-versions", _095_artifact_open_versions_default),
|
|
10952
|
+
pgSql("096-artifact-documents", _096_artifact_documents_default),
|
|
10953
|
+
pgSql("097-artifact-frame-files", _097_artifact_frame_files_default),
|
|
10954
|
+
pgSql("098-artifact-comments", _098_artifact_comments_default),
|
|
10955
|
+
pgSql("099-object-artifact-refs", _099_object_artifact_refs_default),
|
|
10956
|
+
pgSql("100-shared-link-guests", _100_shared_link_guests_default),
|
|
10957
|
+
pgSql("101-connector-connection-owners", _101_connector_connection_owners_default),
|
|
10958
|
+
pgSql("102-user-suspension-source", _102_user_suspension_source_default)
|
|
10424
10959
|
]
|
|
10425
10960
|
});
|
|
10426
10961
|
|
|
@@ -10445,6 +10980,7 @@ function literal(value) {
|
|
|
10445
10980
|
var LOCK_TIMEOUT = "1s";
|
|
10446
10981
|
var LOCK_ATTEMPTS = 120;
|
|
10447
10982
|
var LOCK_RETRY_MS = 1000;
|
|
10983
|
+
var ANALYZE_SCALE_FACTOR = "autovacuum_analyze_scale_factor=0.02";
|
|
10448
10984
|
function objectPartitionName(objectTypeId) {
|
|
10449
10985
|
return `objects_${createHash3("md5").update(objectTypeId).digest("hex").slice(0, 24)}`;
|
|
10450
10986
|
}
|
|
@@ -10452,9 +10988,14 @@ async function ensureObjectPartition(connection, schema, objectTypeId) {
|
|
|
10452
10988
|
const name = objectPartitionName(objectTypeId);
|
|
10453
10989
|
const partition = `${identifier(schema)}.${identifier(name)}`;
|
|
10454
10990
|
const [existing] = await connection`
|
|
10455
|
-
SELECT
|
|
10456
|
-
|
|
10991
|
+
SELECT coalesce(reloptions, '{}') @> ARRAY[${ANALYZE_SCALE_FACTOR}] AS tuned
|
|
10992
|
+
FROM pg_class WHERE oid = to_regclass(${partition})`;
|
|
10993
|
+
if (existing) {
|
|
10994
|
+
if (!existing.tuned) {
|
|
10995
|
+
await connection.unsafe(`ALTER TABLE ${partition} SET (${ANALYZE_SCALE_FACTOR})`);
|
|
10996
|
+
}
|
|
10457
10997
|
return name;
|
|
10998
|
+
}
|
|
10458
10999
|
for (let attempt = 1;; attempt++) {
|
|
10459
11000
|
try {
|
|
10460
11001
|
await createObjectPartition(connection, schema, partition, objectTypeId);
|
|
@@ -10498,7 +11039,7 @@ async function createObjectPartition(connection, schema, partition, objectTypeId
|
|
|
10498
11039
|
code: "55P03"
|
|
10499
11040
|
});
|
|
10500
11041
|
}
|
|
10501
|
-
await connection.unsafe(`CREATE TABLE ${partition} (LIKE ${parent} INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE)`);
|
|
11042
|
+
await connection.unsafe(`CREATE TABLE ${partition} (LIKE ${parent} INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE) WITH (${ANALYZE_SCALE_FACTOR})`);
|
|
10502
11043
|
if (moving)
|
|
10503
11044
|
await moveRowsFromDefault(connection, parent, fallback, partition, type, references);
|
|
10504
11045
|
await connection.unsafe(`ALTER TABLE ${parent} ATTACH PARTITION ${partition} FOR VALUES IN (${type})`);
|
|
@@ -10555,6 +11096,7 @@ import {
|
|
|
10555
11096
|
isVectorProfileQuery,
|
|
10556
11097
|
linkEndpointObjectTypeIds,
|
|
10557
11098
|
normalizeVector,
|
|
11099
|
+
sortTieDirection,
|
|
10558
11100
|
vectorSearchCandidateLimit
|
|
10559
11101
|
} from "@sixb/core/internal/query";
|
|
10560
11102
|
var PAGE_TOKEN_PREFIX = "keyset:";
|
|
@@ -10567,9 +11109,10 @@ function compilePgObjectQuery(projectId, query, options = {}) {
|
|
|
10567
11109
|
let compiled = source.wrapQuery(compileObjectQueryInternal(projectId, query, {
|
|
10568
11110
|
probeLimit: options.includeTotal === false,
|
|
10569
11111
|
correlatedTraversal: options.correlatedTraversal,
|
|
11112
|
+
isolateSortOverflow: options.unorderedStart,
|
|
10570
11113
|
source
|
|
10571
11114
|
}));
|
|
10572
|
-
if (options.
|
|
11115
|
+
if (options.sortIndex) {
|
|
10573
11116
|
let bounded = query;
|
|
10574
11117
|
const expansions = [];
|
|
10575
11118
|
while (bounded.kind === "expand") {
|
|
@@ -10577,17 +11120,19 @@ function compilePgObjectQuery(projectId, query, options = {}) {
|
|
|
10577
11120
|
bounded = bounded.input;
|
|
10578
11121
|
}
|
|
10579
11122
|
if (bounded.kind === "page" || bounded.kind === "limit") {
|
|
10580
|
-
const
|
|
11123
|
+
const { bound, nullRank } = options.sortIndex;
|
|
11124
|
+
const indexed = nullRank ? [`${bound} AND ${nullRank} = 0`, `${bound} AND ${nullRank} = 1`] : [bound];
|
|
11125
|
+
const branches = [...indexed, `NOT (${bound})`].map((startPredicate, branch) => compileObjectQueryInternal(projectId, bounded, {
|
|
10581
11126
|
probeLimit: options.includeTotal === false,
|
|
10582
11127
|
source,
|
|
10583
11128
|
startPredicate,
|
|
10584
|
-
isolateSortOverflow: branch ===
|
|
11129
|
+
isolateSortOverflow: branch === indexed.length,
|
|
10585
11130
|
correlatedTraversal: options.correlatedTraversal
|
|
10586
11131
|
}));
|
|
10587
11132
|
const limit = bounded.kind === "page" ? bounded.pageSize + 1 : bounded.limit + (options.includeTotal === false ? 1 : 0);
|
|
10588
11133
|
let rows = {
|
|
10589
|
-
sql: `SELECT * FROM (
|
|
10590
|
-
args: [...branches
|
|
11134
|
+
sql: `SELECT * FROM (${branches.map((branch) => `(${branch.sql})`).join(" UNION ALL ")}) AS input ORDER BY ${compiled.order.sql} LIMIT ?`,
|
|
11135
|
+
args: [...branches.flatMap((branch) => branch.args), ...compiled.order.args, limit]
|
|
10591
11136
|
};
|
|
10592
11137
|
for (const expansion of expansions) {
|
|
10593
11138
|
rows = expandRows({ ...rows, order: compiled.order }, expansion, {
|
|
@@ -10595,7 +11140,7 @@ function compilePgObjectQuery(projectId, query, options = {}) {
|
|
|
10595
11140
|
source
|
|
10596
11141
|
});
|
|
10597
11142
|
}
|
|
10598
|
-
compiled = { ...compiled, ...rows };
|
|
11143
|
+
compiled = { ...compiled, ...source.wrapStatement(rows.sql, rows.args) };
|
|
10599
11144
|
}
|
|
10600
11145
|
}
|
|
10601
11146
|
const traversal = !options.correlatedTraversal && boundedTraversal(query);
|
|
@@ -10612,6 +11157,9 @@ function compilePgObjectQuery(projectId, query, options = {}) {
|
|
|
10612
11157
|
} : undefined
|
|
10613
11158
|
};
|
|
10614
11159
|
}
|
|
11160
|
+
function linksBetweenVisibleObjects(source) {
|
|
11161
|
+
return source.linksBetweenVisibleObjectsTable ?? source.linksTable;
|
|
11162
|
+
}
|
|
10615
11163
|
function pgObjectsOfType(source, objectTypeId) {
|
|
10616
11164
|
const typed = source.objectsOfType?.(objectTypeId);
|
|
10617
11165
|
if (!typed)
|
|
@@ -11016,7 +11564,7 @@ function compileTraversalRelation(projectId, input, traversal, source, correlate
|
|
|
11016
11564
|
${endpointPredicate ? `AND (${endpointPredicate})` : ""}
|
|
11017
11565
|
AND EXISTS (
|
|
11018
11566
|
SELECT 1 FROM (${input.sql}) AS input
|
|
11019
|
-
JOIN ${source
|
|
11567
|
+
JOIN ${linksBetweenVisibleObjects(source)} AS edge
|
|
11020
11568
|
ON edge.project_id = input.project_id
|
|
11021
11569
|
AND edge.${parent}_type_id = input.object_type_id
|
|
11022
11570
|
AND edge.${parent}_id = input.primary_id
|
|
@@ -11059,7 +11607,7 @@ function compileTraversalProbe(projectId, traversal, source) {
|
|
|
11059
11607
|
const parent = traversal.direction === "incoming" ? "target" : "source";
|
|
11060
11608
|
const limit = 1000;
|
|
11061
11609
|
const statement = compilePgObjectStatement(`SELECT 1 FROM (${input.sql}) AS input
|
|
11062
|
-
JOIN ${source
|
|
11610
|
+
JOIN ${linksBetweenVisibleObjects(source)} AS edge ON edge.project_id = input.project_id
|
|
11063
11611
|
AND edge.${parent}_type_id = input.object_type_id AND edge.${parent}_id = input.primary_id
|
|
11064
11612
|
WHERE edge.link_id = ? ${traversal.direction === "incoming" && traversal.sourceObjectTypeId ? "AND edge.source_type_id = ?" : ""}
|
|
11065
11613
|
LIMIT ?`, [
|
|
@@ -11131,7 +11679,7 @@ function compileExpansionValue(expansion, parent, path, ctx) {
|
|
|
11131
11679
|
whereArgs.push(expansion.sourceObjectTypeId);
|
|
11132
11680
|
}
|
|
11133
11681
|
const from = `
|
|
11134
|
-
FROM ${ctx.source
|
|
11682
|
+
FROM ${linksBetweenVisibleObjects(ctx.source)} AS ${edge}
|
|
11135
11683
|
JOIN ${ctx.source.objectsTable} AS ${target}
|
|
11136
11684
|
ON ${target}.project_id = ${edge}.project_id
|
|
11137
11685
|
AND ${target}.object_type_id = ${neighborType}
|
|
@@ -11191,7 +11739,8 @@ function compileExpansionOrder(expansion, target, neighborType, neighborId) {
|
|
|
11191
11739
|
clauses.push(`CASE WHEN ${jsonTypeExpression(propertyColumn)} IS NULL OR ${jsonTypeExpression(propertyColumn)} = 'null' THEN 1 ELSE 0 END ASC`, `${orderedValueExpression(field.scalarKind, propertyColumn)} ${direction}`);
|
|
11192
11740
|
args.push(field.propertyId, field.propertyId, field.propertyId);
|
|
11193
11741
|
}
|
|
11194
|
-
|
|
11742
|
+
const ties = sortTieDirection(expansion.orderBy ?? []).toUpperCase();
|
|
11743
|
+
clauses.push(`${neighborType} ${ties}`, `${neighborId} ${ties}`);
|
|
11195
11744
|
return { sql: clauses.join(", "), args };
|
|
11196
11745
|
}
|
|
11197
11746
|
function compileSet(projectId, op, inputs, ctx) {
|
|
@@ -11368,7 +11917,7 @@ function compileAggregateTraversal(projectId, traversal, source) {
|
|
|
11368
11917
|
FROM (
|
|
11369
11918
|
SELECT DISTINCT edge.source_id AS primary_id
|
|
11370
11919
|
FROM (${input.sql}) AS input
|
|
11371
|
-
JOIN ${source
|
|
11920
|
+
JOIN ${linksBetweenVisibleObjects(source)} AS edge ON edge.project_id = input.project_id
|
|
11372
11921
|
AND edge.target_type_id = input.object_type_id AND edge.target_id = input.primary_id
|
|
11373
11922
|
WHERE edge.link_id = ? AND edge.source_type_id = ?
|
|
11374
11923
|
) AS endpoints
|
|
@@ -11695,13 +12244,13 @@ function sortOrderFields(fields) {
|
|
|
11695
12244
|
scalarKind: field.scalarKind
|
|
11696
12245
|
});
|
|
11697
12246
|
}
|
|
11698
|
-
orderFields.push(...identityOrderFields());
|
|
12247
|
+
orderFields.push(...identityOrderFields(sortTieDirection(fields)));
|
|
11699
12248
|
return orderFields;
|
|
11700
12249
|
}
|
|
11701
|
-
function identityOrderFields() {
|
|
12250
|
+
function identityOrderFields(direction = "asc") {
|
|
11702
12251
|
return [
|
|
11703
|
-
{ kind: "column", column: "object_type_id", direction
|
|
11704
|
-
{ kind: "column", column: "primary_id", direction
|
|
12252
|
+
{ kind: "column", column: "object_type_id", direction },
|
|
12253
|
+
{ kind: "column", column: "primary_id", direction }
|
|
11705
12254
|
];
|
|
11706
12255
|
}
|
|
11707
12256
|
function compileKeysetRanges(fields, cursor, propertyColumn) {
|
|
@@ -12020,6 +12569,7 @@ async function createQueryIndex(connection, schema, definition) {
|
|
|
12020
12569
|
}
|
|
12021
12570
|
return name;
|
|
12022
12571
|
}
|
|
12572
|
+
var STATISTICS_KINDS = "mcv";
|
|
12023
12573
|
async function ensureExpressionStatistics(connection, schema, indexes) {
|
|
12024
12574
|
const expressionsByType = new Map;
|
|
12025
12575
|
for (const index of indexes) {
|
|
@@ -12032,21 +12582,33 @@ async function ensureExpressionStatistics(connection, schema, indexes) {
|
|
|
12032
12582
|
}
|
|
12033
12583
|
for (const [objectTypeId, expressions] of expressionsByType) {
|
|
12034
12584
|
const partition = `${identifier(schema)}.${identifier(objectPartitionName(objectTypeId))}`;
|
|
12035
|
-
|
|
12036
|
-
|
|
12037
|
-
const
|
|
12038
|
-
|
|
12039
|
-
|
|
12040
|
-
|
|
12041
|
-
|
|
12042
|
-
|
|
12043
|
-
|
|
12044
|
-
|
|
12045
|
-
|
|
12046
|
-
|
|
12047
|
-
|
|
12585
|
+
await runPgTransactionOn(connection, async () => {
|
|
12586
|
+
let statisticsChanged = false;
|
|
12587
|
+
for (const expression of expressions) {
|
|
12588
|
+
const name = statisticsName(objectTypeId, `${STATISTICS_KINDS}:${expression}`);
|
|
12589
|
+
if (await statisticsExist(connection, schema, name))
|
|
12590
|
+
continue;
|
|
12591
|
+
const superseded = statisticsName(objectTypeId, expression);
|
|
12592
|
+
if (await statisticsExist(connection, schema, superseded)) {
|
|
12593
|
+
await connection.unsafe(`DROP STATISTICS ${identifier(schema)}.${identifier(superseded)}`);
|
|
12594
|
+
}
|
|
12595
|
+
await connection.unsafe(`CREATE STATISTICS ${identifier(schema)}.${identifier(name)} (${STATISTICS_KINDS}) ON project_id, ${expression} FROM ${partition}`);
|
|
12596
|
+
statisticsChanged = true;
|
|
12597
|
+
}
|
|
12598
|
+
if (statisticsChanged)
|
|
12599
|
+
await connection.unsafe(`ANALYZE ${partition}`);
|
|
12600
|
+
});
|
|
12048
12601
|
}
|
|
12049
12602
|
}
|
|
12603
|
+
function statisticsName(objectTypeId, definition) {
|
|
12604
|
+
return `sixb_query_stats_${createHash4("sha256").update(`${objectTypeId}:${definition}`).digest("hex").slice(0, 24)}`;
|
|
12605
|
+
}
|
|
12606
|
+
async function statisticsExist(connection, schema, name) {
|
|
12607
|
+
const [existing] = await connection`
|
|
12608
|
+
SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
|
|
12609
|
+
WHERE n.nspname = ${schema} AND s.stxname = ${name}`;
|
|
12610
|
+
return existing !== undefined;
|
|
12611
|
+
}
|
|
12050
12612
|
function statisticsExpressions(index) {
|
|
12051
12613
|
const equality = new Set(Object.keys(index.where ?? {}));
|
|
12052
12614
|
if (index.kind === "filter")
|
|
@@ -12109,13 +12671,16 @@ function compileIndex(index) {
|
|
|
12109
12671
|
fields.push(`(properties ->> ${literal(propertyId)})`);
|
|
12110
12672
|
}
|
|
12111
12673
|
for (const field of index.fields) {
|
|
12112
|
-
|
|
12113
|
-
fields.push(`(CASE WHEN jsonb_typeof(${property}) IS NULL OR jsonb_typeof(${property}) = 'null' THEN 1 ELSE 0 END)`, `${sortValue(field)} ${field.direction === "desc" ? "DESC" : "ASC"}`);
|
|
12674
|
+
fields.push(sortIndexNullRank(field.propertyId), `${sortValue(field)} ${field.direction === "desc" ? "DESC" : "ASC"}`);
|
|
12114
12675
|
}
|
|
12115
|
-
fields.push("primary_id");
|
|
12676
|
+
fields.push(index.fields.at(-1)?.direction === "desc" ? "primary_id DESC" : "primary_id");
|
|
12116
12677
|
}
|
|
12117
12678
|
return `(${fields.join(", ")}) WHERE ${predicate.join(" AND ")}`;
|
|
12118
12679
|
}
|
|
12680
|
+
function sortIndexNullRank(propertyId) {
|
|
12681
|
+
const property = `properties -> ${literal(propertyId)}`;
|
|
12682
|
+
return `(CASE WHEN jsonb_typeof(${property}) IS NULL OR jsonb_typeof(${property}) = 'null' THEN 1 ELSE 0 END)`;
|
|
12683
|
+
}
|
|
12119
12684
|
function sortValue(field) {
|
|
12120
12685
|
const text = `(properties ->> ${literal(field.propertyId)})`;
|
|
12121
12686
|
if (field.scalarKind === "decimal")
|
|
@@ -12177,7 +12742,7 @@ async function readPgSortIndexes(sql) {
|
|
|
12177
12742
|
}
|
|
12178
12743
|
return indexes;
|
|
12179
12744
|
}
|
|
12180
|
-
async function
|
|
12745
|
+
async function findPgSortIndexRead(sql, catalog, projectId, query) {
|
|
12181
12746
|
while (query.kind === "expand")
|
|
12182
12747
|
query = query.input;
|
|
12183
12748
|
if (query.kind !== "page" && query.kind !== "limit")
|
|
@@ -12219,12 +12784,22 @@ async function findPgSortIndexBound(sql, catalog, projectId, query) {
|
|
|
12219
12784
|
continue;
|
|
12220
12785
|
if (index.equality !== undefined && (!Array.isArray(index.equality) || !index.equality.every((property) => typeof property === "string" && equality.has(property))))
|
|
12221
12786
|
continue;
|
|
12222
|
-
if (!index.fields.every((field, i) => field && typeof field.propertyId === "string" && fields[i]?.kind === "property" && fields[i].propertyId === field.propertyId &&
|
|
12787
|
+
if (!index.fields.every((field, i) => field && typeof field.propertyId === "string" && fields[i]?.kind === "property" && fields[i].propertyId === field.propertyId && field.scalarKind === "decimal" === (fields[i].scalarKind === "decimal") && isTextScalarKind(field.scalarKind) === isTextScalarKind(fields[i].scalarKind)))
|
|
12223
12788
|
continue;
|
|
12224
|
-
|
|
12225
|
-
|
|
12226
|
-
|
|
12227
|
-
|
|
12789
|
+
const forward = index.fields.every((field, i) => (field.direction ?? "asc") === (fields[i]?.direction ?? "asc"));
|
|
12790
|
+
if (forward)
|
|
12791
|
+
matches.push({ index, backwards: false });
|
|
12792
|
+
else if (index.fields.length === 1)
|
|
12793
|
+
matches.push({ index, backwards: true });
|
|
12794
|
+
}
|
|
12795
|
+
matches.sort((a, b) => (b.index.equality?.length ?? 0) - (a.index.equality?.length ?? 0) || Number(a.backwards) - Number(b.backwards));
|
|
12796
|
+
const [match] = matches;
|
|
12797
|
+
if (!match)
|
|
12798
|
+
return;
|
|
12799
|
+
const bound = sortIndexBound(match.index);
|
|
12800
|
+
if (!match.backwards)
|
|
12801
|
+
return { objectTypeId, bound };
|
|
12802
|
+
return { objectTypeId, bound, nullRank: sortIndexNullRank(match.index.fields[0].propertyId) };
|
|
12228
12803
|
}
|
|
12229
12804
|
// src/objects/storage.ts
|
|
12230
12805
|
import { hasVectorProfile as hasVectorProfile4 } from "@sixb/core/internal/query";
|
|
@@ -12257,6 +12832,82 @@ class PgAggregateAdmission {
|
|
|
12257
12832
|
// src/objects/conditional-reader.ts
|
|
12258
12833
|
import { hasVectorProfile as hasVectorProfile3 } from "@sixb/core/internal/query";
|
|
12259
12834
|
|
|
12835
|
+
// src/objects/visibility.ts
|
|
12836
|
+
function compilePgVisibilityKeys(keys) {
|
|
12837
|
+
switch (keys.kind) {
|
|
12838
|
+
case "everyone":
|
|
12839
|
+
return `CASE WHEN ${predicateSql(keys.when)} THEN ARRAY[${text(EVERYONE)}]::text[] ELSE ${NO_KEYS} END`;
|
|
12840
|
+
case "user": {
|
|
12841
|
+
const ref = propertySql(keys.propertyId);
|
|
12842
|
+
return `CASE WHEN jsonb_typeof(${ref}) = 'object' AND (${ref} - 'type' - 'id') = '{}'::jsonb AND ${ref} -> 'type' = '"user"'::jsonb AND jsonb_typeof(${ref} -> 'id') = 'string' AND char_length(${ref} ->> 'id') <= ${MAX_KEYED_ID_LENGTH} THEN ARRAY[${text(USER_PREFIX)} || ${withoutEscapes(`left(substr((${ref} -> 'id')::text, 2), -1)`)}] ELSE ${NO_KEYS} END`;
|
|
12843
|
+
}
|
|
12844
|
+
case "users": {
|
|
12845
|
+
const users = `@.type == "user" && @.id.type() == "string"`;
|
|
12846
|
+
const keysOf = (filter) => {
|
|
12847
|
+
const path = text(`strict $[*] ? (${filter}).id`);
|
|
12848
|
+
const ids = withoutEscapes(`jsonb_path_query_array(${propertySql(keys.propertyId)}, ${path}, '{}', true)::text`);
|
|
12849
|
+
const separator = `E'\\x1f'`;
|
|
12850
|
+
return `coalesce(string_to_array(${text(USER_PREFIX)} || replace(NULLIF(left(substr(${ids}, 3), -2), ''), '", "', ${separator} || ${text(USER_PREFIX)}), ${separator}), ${NO_KEYS})`;
|
|
12851
|
+
};
|
|
12852
|
+
const bounded = `${users} && @.id like_regex "^.{0,${MAX_KEYED_ID_LENGTH}}$" flag "s"`;
|
|
12853
|
+
return `CASE WHEN octet_length(${propertySql(keys.propertyId)}::text) <= ${SHORT_LIST_BYTES} THEN ${keysOf(users)} ELSE ${keysOf(bounded)} END`;
|
|
12854
|
+
}
|
|
12855
|
+
case "union":
|
|
12856
|
+
return `(${keys.items.map((item) => `(${compilePgVisibilityKeys(item)})`).join(" || ")})`;
|
|
12857
|
+
case "when":
|
|
12858
|
+
return `CASE WHEN ${predicateSql(keys.when)} THEN ${compilePgVisibilityKeys(keys.keys)} ELSE ${NO_KEYS} END`;
|
|
12859
|
+
}
|
|
12860
|
+
}
|
|
12861
|
+
function pgVisibilityReaderKeys(user) {
|
|
12862
|
+
if (!user)
|
|
12863
|
+
return [EVERYONE];
|
|
12864
|
+
const id = JSON.stringify(user.id).slice(1, -1).replaceAll("\\\\", ESCAPED_BACKSLASH).replaceAll("\\\"", ESCAPED_QUOTE);
|
|
12865
|
+
return [EVERYONE, `${USER_PREFIX}${id}`];
|
|
12866
|
+
}
|
|
12867
|
+
function compilePgConditionFilter(condition) {
|
|
12868
|
+
const visibility = condition.visibility;
|
|
12869
|
+
if (!visibility || (visibility.user?.id.length ?? 0) > MAX_KEYED_ID_LENGTH) {
|
|
12870
|
+
return compilePredicate(condition.predicate);
|
|
12871
|
+
}
|
|
12872
|
+
const readerKeys = pgVisibilityReaderKeys(visibility.user);
|
|
12873
|
+
const clauses = visibility.keys.map((keys) => `((${compilePgVisibilityKeys(keys)}) && ?::text[])`);
|
|
12874
|
+
const args = visibility.keys.map(() => readerKeys);
|
|
12875
|
+
if (visibility.remainder) {
|
|
12876
|
+
const remainder = compilePredicate(visibility.remainder);
|
|
12877
|
+
clauses.push(remainder.sql);
|
|
12878
|
+
args.push(...remainder.args);
|
|
12879
|
+
}
|
|
12880
|
+
return { sql: `(${clauses.join(" OR ")})`, args };
|
|
12881
|
+
}
|
|
12882
|
+
var MAX_KEYED_ID_LENGTH = 200;
|
|
12883
|
+
var ESCAPED_BACKSLASH = "\x1E";
|
|
12884
|
+
var ESCAPED_QUOTE = "\x1D";
|
|
12885
|
+
var SHORT_LIST_BYTES = 2000;
|
|
12886
|
+
var EVERYONE = "*";
|
|
12887
|
+
var USER_PREFIX = "u:";
|
|
12888
|
+
var NO_KEYS = "'{}'::text[]";
|
|
12889
|
+
function propertySql(propertyId) {
|
|
12890
|
+
return `(properties -> ${text(propertyId)}::text)`;
|
|
12891
|
+
}
|
|
12892
|
+
function withoutEscapes(printed) {
|
|
12893
|
+
return `replace(replace(${printed}, ${text("\\\\")}, E'\\x1e'), ${text("\\\"")}, E'\\x1d')`;
|
|
12894
|
+
}
|
|
12895
|
+
function text(value) {
|
|
12896
|
+
return literal(value).replace(/\?/g, "\\x3f");
|
|
12897
|
+
}
|
|
12898
|
+
function predicateSql(predicate) {
|
|
12899
|
+
const compiled = compilePredicate(predicate);
|
|
12900
|
+
let index = 0;
|
|
12901
|
+
return `(${compiled.sql.replace(/\?/g, () => inlineValue(compiled.args[index++]))})`;
|
|
12902
|
+
}
|
|
12903
|
+
function inlineValue(value) {
|
|
12904
|
+
if (typeof value === "string")
|
|
12905
|
+
return text(value);
|
|
12906
|
+
if (typeof value === "number" && Number.isFinite(value))
|
|
12907
|
+
return `(${String(value)})`;
|
|
12908
|
+
throw new Error(`[SixbPg] A view condition compares a value it cannot index: ${String(value)}.`);
|
|
12909
|
+
}
|
|
12910
|
+
|
|
12260
12911
|
// src/objects/read-scope.ts
|
|
12261
12912
|
function compilePgSelectedObjectReadSource(projectId, scope, maxTraversalFacts) {
|
|
12262
12913
|
const scopeJson = JSON.stringify({
|
|
@@ -12264,8 +12915,9 @@ function compilePgSelectedObjectReadSource(projectId, scope, maxTraversalFacts)
|
|
|
12264
12915
|
objects: scope.objects,
|
|
12265
12916
|
steps: scope.steps
|
|
12266
12917
|
});
|
|
12918
|
+
const relations = compileSelectedReadScopeCte(scopeJson, projectId);
|
|
12267
12919
|
const source = {
|
|
12268
|
-
...scopedQuerySource(
|
|
12920
|
+
...scopedQuerySource(() => relations),
|
|
12269
12921
|
objectPropertyPermissionsTable: "_sixb_scope_object_permissions",
|
|
12270
12922
|
traversalProbe: { sql: "", args: [] }
|
|
12271
12923
|
};
|
|
@@ -12288,56 +12940,84 @@ function compilePgSelectedObjectReadSource(projectId, scope, maxTraversalFacts)
|
|
|
12288
12940
|
function compilePgConditionalObjectReadSource(projectId, scope) {
|
|
12289
12941
|
const filters = new Map(scope.conditions.map((condition) => [
|
|
12290
12942
|
condition.objectTypeId,
|
|
12291
|
-
|
|
12943
|
+
compilePgConditionFilter(condition)
|
|
12292
12944
|
]));
|
|
12945
|
+
const relations = compileConditionalReadScopeRelations(projectId, filters);
|
|
12293
12946
|
return Object.freeze({
|
|
12294
|
-
...scopedQuerySource(
|
|
12947
|
+
...scopedQuerySource((sql) => withReadRelations(relations, sql)),
|
|
12948
|
+
linksBetweenVisibleObjectsTable: "links",
|
|
12295
12949
|
objectsOfType: (objectTypeId) => {
|
|
12296
12950
|
const filter = filters.get(objectTypeId);
|
|
12297
12951
|
return filter ? { table: "objects", filter } : { table: "objects" };
|
|
12298
12952
|
}
|
|
12299
12953
|
});
|
|
12300
12954
|
}
|
|
12301
|
-
function scopedQuerySource(
|
|
12302
|
-
const wrapStatement = (sql, args = []) =>
|
|
12303
|
-
|
|
12955
|
+
function scopedQuerySource(withClause) {
|
|
12956
|
+
const wrapStatement = (sql, args = []) => {
|
|
12957
|
+
const compiled = withClause(sql);
|
|
12958
|
+
return {
|
|
12959
|
+
sql: `${compiled.sql}
|
|
12304
12960
|
${sql}`,
|
|
12305
|
-
|
|
12306
|
-
|
|
12961
|
+
args: [...compiled.args, ...args]
|
|
12962
|
+
};
|
|
12963
|
+
};
|
|
12307
12964
|
return {
|
|
12308
12965
|
objectsTable: "_sixb_scope_objects",
|
|
12309
12966
|
linksTable: "_sixb_scope_links",
|
|
12310
12967
|
wrapStatement,
|
|
12311
|
-
wrapQuery: (query) =>
|
|
12312
|
-
|
|
12313
|
-
|
|
12314
|
-
|
|
12315
|
-
|
|
12316
|
-
|
|
12317
|
-
|
|
12318
|
-
|
|
12319
|
-
|
|
12320
|
-
|
|
12321
|
-
|
|
12322
|
-
|
|
12323
|
-
|
|
12324
|
-
|
|
12968
|
+
wrapQuery: (query) => {
|
|
12969
|
+
const total = wrapStatement(query.totalSql, query.totalArgs);
|
|
12970
|
+
return {
|
|
12971
|
+
...query,
|
|
12972
|
+
...wrapStatement(query.sql, query.args),
|
|
12973
|
+
vectorProbe: query.vectorProbe ? {
|
|
12974
|
+
...query.vectorProbe,
|
|
12975
|
+
...wrapStatement(query.vectorProbe.sql, query.vectorProbe.args)
|
|
12976
|
+
} : undefined,
|
|
12977
|
+
totalSql: total.sql,
|
|
12978
|
+
totalArgs: total.args,
|
|
12979
|
+
...query.hasMoreProbe ? {
|
|
12980
|
+
hasMoreProbe: {
|
|
12981
|
+
...query.hasMoreProbe,
|
|
12982
|
+
...wrapStatement(query.hasMoreProbe.sql, query.hasMoreProbe.args)
|
|
12983
|
+
}
|
|
12984
|
+
} : {}
|
|
12985
|
+
};
|
|
12986
|
+
}
|
|
12987
|
+
};
|
|
12988
|
+
}
|
|
12989
|
+
function withReadRelations(relations, statementSql) {
|
|
12990
|
+
const read = relations.filter((relation) => new RegExp(`\\b${relation.name}\\b`).test(statementSql));
|
|
12991
|
+
if (read.length === 0)
|
|
12992
|
+
return { sql: "", args: [] };
|
|
12993
|
+
return {
|
|
12994
|
+
sql: `WITH ${read.map((relation) => relation.sql).join(`,
|
|
12995
|
+
`)}`,
|
|
12996
|
+
args: read.flatMap((relation) => relation.args)
|
|
12325
12997
|
};
|
|
12326
12998
|
}
|
|
12327
|
-
function
|
|
12999
|
+
function compileConditionalReadScopeRelations(projectId, filters) {
|
|
12328
13000
|
if (filters.size === 0) {
|
|
12329
|
-
return
|
|
12330
|
-
|
|
12331
|
-
|
|
12332
|
-
|
|
12333
|
-
|
|
12334
|
-
|
|
12335
|
-
|
|
12336
|
-
|
|
12337
|
-
|
|
12338
|
-
|
|
12339
|
-
|
|
12340
|
-
|
|
13001
|
+
return [
|
|
13002
|
+
{
|
|
13003
|
+
name: "_sixb_scope_objects",
|
|
13004
|
+
sql: `
|
|
13005
|
+
_sixb_scope_objects AS NOT MATERIALIZED (
|
|
13006
|
+
SELECT * FROM objects WHERE project_id = ?
|
|
13007
|
+
)
|
|
13008
|
+
`,
|
|
13009
|
+
args: [projectId]
|
|
13010
|
+
},
|
|
13011
|
+
{
|
|
13012
|
+
name: "_sixb_scope_links",
|
|
13013
|
+
sql: `
|
|
13014
|
+
_sixb_scope_links AS NOT MATERIALIZED (
|
|
13015
|
+
SELECT * FROM links WHERE project_id = ?
|
|
13016
|
+
)
|
|
13017
|
+
`,
|
|
13018
|
+
args: [projectId]
|
|
13019
|
+
}
|
|
13020
|
+
];
|
|
12341
13021
|
}
|
|
12342
13022
|
const conditionedTypeArgs = [...filters.keys()];
|
|
12343
13023
|
const conditionedTypes = conditionedTypeArgs.map(() => "?::text").join(", ");
|
|
@@ -12347,51 +13027,66 @@ function compileConditionalReadScopeCte(projectId, filters) {
|
|
|
12347
13027
|
matches.push(`(object_type_id = ?::text AND ${filter.sql})`);
|
|
12348
13028
|
matchArgs.push(objectTypeId, ...filter.args);
|
|
12349
13029
|
}
|
|
12350
|
-
const visibleEndpoint = (end) =>
|
|
12351
|
-
|
|
12352
|
-
|
|
12353
|
-
|
|
12354
|
-
|
|
12355
|
-
|
|
12356
|
-
|
|
12357
|
-
AND
|
|
12358
|
-
|
|
12359
|
-
|
|
12360
|
-
|
|
12361
|
-
|
|
13030
|
+
const visibleEndpoint = (end) => {
|
|
13031
|
+
const checks = [];
|
|
13032
|
+
const args = [...conditionedTypeArgs];
|
|
13033
|
+
for (const [objectTypeId, filter] of filters) {
|
|
13034
|
+
checks.push(`
|
|
13035
|
+
(
|
|
13036
|
+
stored.${end}_type_id = ?::text
|
|
13037
|
+
AND EXISTS (
|
|
13038
|
+
SELECT 1
|
|
13039
|
+
FROM objects AS endpoint
|
|
13040
|
+
WHERE endpoint.project_id = stored.project_id
|
|
13041
|
+
AND endpoint.object_type_id = ?::text
|
|
13042
|
+
AND endpoint.primary_id = stored.${end}_id
|
|
13043
|
+
AND (${filter.sql})
|
|
13044
|
+
)
|
|
13045
|
+
)
|
|
13046
|
+
`);
|
|
13047
|
+
args.push(objectTypeId, objectTypeId, ...filter.args);
|
|
13048
|
+
}
|
|
13049
|
+
return {
|
|
13050
|
+
sql: `(stored.${end}_type_id NOT IN (${conditionedTypes}) OR ${checks.join(" OR ")})`,
|
|
13051
|
+
args
|
|
13052
|
+
};
|
|
13053
|
+
};
|
|
13054
|
+
const visibleSource = visibleEndpoint("source");
|
|
13055
|
+
const visibleTarget = visibleEndpoint("target");
|
|
13056
|
+
return [
|
|
13057
|
+
{
|
|
13058
|
+
name: "_sixb_scope_objects",
|
|
13059
|
+
sql: `
|
|
13060
|
+
_sixb_scope_objects AS NOT MATERIALIZED (
|
|
13061
|
+
SELECT *
|
|
13062
|
+
FROM objects
|
|
13063
|
+
WHERE project_id = ?
|
|
13064
|
+
AND (object_type_id NOT IN (${conditionedTypes}) OR ${matches.join(" OR ")})
|
|
13065
|
+
)
|
|
13066
|
+
`,
|
|
13067
|
+
args: [projectId, ...conditionedTypeArgs, ...matchArgs]
|
|
13068
|
+
},
|
|
13069
|
+
{
|
|
13070
|
+
name: "_sixb_scope_links",
|
|
13071
|
+
sql: `
|
|
13072
|
+
_sixb_scope_links AS NOT MATERIALIZED (
|
|
13073
|
+
SELECT stored.*
|
|
13074
|
+
FROM links AS stored
|
|
13075
|
+
WHERE stored.project_id = ?
|
|
13076
|
+
AND ${visibleSource.sql}
|
|
13077
|
+
AND ${visibleTarget.sql}
|
|
13078
|
+
)
|
|
13079
|
+
`,
|
|
13080
|
+
args: [projectId, ...visibleSource.args, ...visibleTarget.args]
|
|
13081
|
+
}
|
|
13082
|
+
];
|
|
13083
|
+
}
|
|
13084
|
+
function compileSelectedReadScopeCte(scopeJson, projectId) {
|
|
12362
13085
|
return {
|
|
12363
13086
|
sql: `
|
|
12364
|
-
WITH
|
|
12365
|
-
|
|
12366
|
-
SELECT
|
|
12367
|
-
FROM objects
|
|
12368
|
-
WHERE project_id = ?
|
|
12369
|
-
AND (object_type_id NOT IN (${conditionedTypes}) OR ${matches.join(" OR ")})
|
|
12370
|
-
),
|
|
12371
|
-
_sixb_scope_links AS NOT MATERIALIZED (
|
|
12372
|
-
SELECT stored.*
|
|
12373
|
-
FROM links AS stored
|
|
12374
|
-
WHERE stored.project_id = ?
|
|
12375
|
-
AND ${visibleEndpoint("source")}
|
|
12376
|
-
AND ${visibleEndpoint("target")}
|
|
12377
|
-
)
|
|
12378
|
-
`,
|
|
12379
|
-
args: [
|
|
12380
|
-
projectId,
|
|
12381
|
-
...conditionedTypeArgs,
|
|
12382
|
-
...matchArgs,
|
|
12383
|
-
projectId,
|
|
12384
|
-
...conditionedTypeArgs,
|
|
12385
|
-
...conditionedTypeArgs
|
|
12386
|
-
]
|
|
12387
|
-
};
|
|
12388
|
-
}
|
|
12389
|
-
function compileSelectedReadScopeCte(scopeJson, projectId) {
|
|
12390
|
-
return {
|
|
12391
|
-
sql: `
|
|
12392
|
-
WITH RECURSIVE
|
|
12393
|
-
_sixb_scope_document(scope) AS (
|
|
12394
|
-
SELECT ?::text::jsonb
|
|
13087
|
+
WITH RECURSIVE
|
|
13088
|
+
_sixb_scope_document(scope) AS (
|
|
13089
|
+
SELECT ?::text::jsonb
|
|
12395
13090
|
),
|
|
12396
13091
|
_sixb_scope_roots(root_id, node_id, object_type_id, primary_id) AS (
|
|
12397
13092
|
SELECT
|
|
@@ -13010,11 +13705,13 @@ class PgObjectReader {
|
|
|
13010
13705
|
source;
|
|
13011
13706
|
aggregateAdmission;
|
|
13012
13707
|
sortIndexes;
|
|
13013
|
-
|
|
13708
|
+
readsFewObjects;
|
|
13709
|
+
constructor(sql, source, aggregateAdmission, sortIndexes, readsFewObjects) {
|
|
13014
13710
|
this.sql = sql;
|
|
13015
13711
|
this.source = source;
|
|
13016
13712
|
this.aggregateAdmission = aggregateAdmission;
|
|
13017
13713
|
this.sortIndexes = sortIndexes;
|
|
13714
|
+
this.readsFewObjects = readsFewObjects;
|
|
13018
13715
|
}
|
|
13019
13716
|
async assertVectorSearch(params) {
|
|
13020
13717
|
if (!hasVectorProfile2(params.query))
|
|
@@ -13054,9 +13751,13 @@ class PgObjectReader {
|
|
|
13054
13751
|
async compilePageQuery(params) {
|
|
13055
13752
|
const sql = this.sql;
|
|
13056
13753
|
const source = this.source;
|
|
13057
|
-
|
|
13754
|
+
let sortIndex = this.sortIndexes ? await findPgSortIndexRead(sql, this.sortIndexes, params.projectId, params.query) : undefined;
|
|
13755
|
+
const unorderedStart = sortIndex !== undefined && await this.readsFewObjects?.(sql, sortIndex.objectTypeId) === true;
|
|
13756
|
+
if (unorderedStart)
|
|
13757
|
+
sortIndex = undefined;
|
|
13058
13758
|
let compiled = compilePgObjectQuery(params.projectId, params.query, {
|
|
13059
|
-
|
|
13759
|
+
sortIndex,
|
|
13760
|
+
unorderedStart,
|
|
13060
13761
|
includeTotal: params.includeTotal,
|
|
13061
13762
|
source
|
|
13062
13763
|
});
|
|
@@ -13068,7 +13769,8 @@ class PgObjectReader {
|
|
|
13068
13769
|
includeTotal: params.includeTotal,
|
|
13069
13770
|
source,
|
|
13070
13771
|
correlatedTraversal: true,
|
|
13071
|
-
|
|
13772
|
+
sortIndex,
|
|
13773
|
+
unorderedStart
|
|
13072
13774
|
});
|
|
13073
13775
|
}
|
|
13074
13776
|
}
|
|
@@ -13231,16 +13933,29 @@ class PgObjectReader {
|
|
|
13231
13933
|
const result = new Map;
|
|
13232
13934
|
if (params.items.length === 0)
|
|
13233
13935
|
return result;
|
|
13234
|
-
const
|
|
13235
|
-
const
|
|
13936
|
+
const requestedByType = new Map;
|
|
13937
|
+
for (const [batchIndex, item] of params.items.entries()) {
|
|
13938
|
+
const requested = requestedByType.get(item.objectTypeId) ?? [];
|
|
13939
|
+
requested.push({ primaryId: item.primaryId, batchIndex });
|
|
13940
|
+
requestedByType.set(item.objectTypeId, requested);
|
|
13941
|
+
}
|
|
13942
|
+
const branches = [];
|
|
13943
|
+
const args = [];
|
|
13944
|
+
for (const [objectTypeId, requested] of requestedByType) {
|
|
13945
|
+
const objects = pgObjectsOfType(source, objectTypeId);
|
|
13946
|
+
branches.push(`
|
|
13236
13947
|
SELECT object.*, requested."batchIndex" AS _batch_index
|
|
13237
13948
|
FROM jsonb_to_recordset(?::text::jsonb)
|
|
13238
|
-
AS requested("
|
|
13239
|
-
JOIN ${
|
|
13949
|
+
AS requested("primaryId" text, "batchIndex" integer)
|
|
13950
|
+
JOIN ${objects.table} AS object
|
|
13240
13951
|
ON object.project_id = ?
|
|
13241
|
-
AND object.object_type_id =
|
|
13952
|
+
AND object.object_type_id = ?::text
|
|
13242
13953
|
AND object.primary_id = requested."primaryId"
|
|
13243
|
-
|
|
13954
|
+
${objects.filter}
|
|
13955
|
+
`);
|
|
13956
|
+
args.push(JSON.stringify(requested), params.projectId, objectTypeId, ...objects.args);
|
|
13957
|
+
}
|
|
13958
|
+
const statement = compilePgObjectStatement(branches.join(" UNION ALL "), args, source);
|
|
13244
13959
|
const rows = await sql.unsafe(statement.sql, statement.args);
|
|
13245
13960
|
const rowsByIndex = new Map(rows.map((row) => [row._batch_index, row]));
|
|
13246
13961
|
for (const [index, item] of params.items.entries()) {
|
|
@@ -13592,13 +14307,186 @@ function createPgScopedReader(params, queryCapabilities) {
|
|
|
13592
14307
|
});
|
|
13593
14308
|
}
|
|
13594
14309
|
|
|
14310
|
+
// src/objects/visibility-indexes.ts
|
|
14311
|
+
import { createHash as createHash6 } from "crypto";
|
|
14312
|
+
async function preparePgVisibilityIndexes(connection, schema, plan) {
|
|
14313
|
+
if (plan.objectTypes.every((type) => type.visibility === undefined)) {
|
|
14314
|
+
return { names: [], warnings: [] };
|
|
14315
|
+
}
|
|
14316
|
+
const warnings = [];
|
|
14317
|
+
const built = [];
|
|
14318
|
+
for (const index of visibilityIndexes(plan)) {
|
|
14319
|
+
const warning = await createVisibilityIndex(connection, schema, plan.projectId, index);
|
|
14320
|
+
if (warning)
|
|
14321
|
+
warnings.push(warning);
|
|
14322
|
+
else
|
|
14323
|
+
built.push(index);
|
|
14324
|
+
}
|
|
14325
|
+
await ensureVisibilityStatistics(connection, schema, built);
|
|
14326
|
+
await dropUnusedVisibilityIndexes(connection, schema, plan.projectId, built);
|
|
14327
|
+
return { names: built.map((index) => index.name), warnings };
|
|
14328
|
+
}
|
|
14329
|
+
function visibilityIndexes(plan) {
|
|
14330
|
+
return plan.objectTypes.flatMap((type) => {
|
|
14331
|
+
const partition = objectPartitionName(type.objectTypeId);
|
|
14332
|
+
return (type.visibility ?? []).map((keys) => {
|
|
14333
|
+
const expression = compilePgVisibilityKeys(keys);
|
|
14334
|
+
const statistics = visibilityStatisticsName(type.objectTypeId, keys);
|
|
14335
|
+
const name = `sixb_visibility_${digest(`${partition}:${plan.projectId}:${expression}`, 32)}`;
|
|
14336
|
+
return { objectTypeId: type.objectTypeId, partition, expression, name, statistics };
|
|
14337
|
+
});
|
|
14338
|
+
});
|
|
14339
|
+
}
|
|
14340
|
+
function visibilityStatisticsName(objectTypeId, keys) {
|
|
14341
|
+
const expression = compilePgVisibilityKeys(keys);
|
|
14342
|
+
return `sixb_visibility_stats_${digest(`${objectPartitionName(objectTypeId)}:${expression}`, 24)}`;
|
|
14343
|
+
}
|
|
14344
|
+
async function createVisibilityIndex(connection, schema, projectId, index) {
|
|
14345
|
+
const table = `${identifier(schema)}.${identifier(index.partition)}`;
|
|
14346
|
+
try {
|
|
14347
|
+
await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(index.name)} ON ${table} USING gin ((${index.expression})) WHERE project_id = ${literal(projectId)}`);
|
|
14348
|
+
} catch (error) {
|
|
14349
|
+
if (error.code !== FUNCTION_NOT_IMMUTABLE)
|
|
14350
|
+
throw error;
|
|
14351
|
+
return conditionNotIndexed(index.objectTypeId, error);
|
|
14352
|
+
}
|
|
14353
|
+
const [built] = await connection`
|
|
14354
|
+
SELECT i.indisvalid AS valid FROM pg_index i
|
|
14355
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
14356
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
14357
|
+
WHERE n.nspname = ${schema} AND c.relname = ${index.name}`;
|
|
14358
|
+
if (!built?.valid) {
|
|
14359
|
+
throw new Error(`[SixbPg] Visibility index ${schema}.${index.name} is invalid after an interrupted build. Drop that index concurrently and prepare again.`);
|
|
14360
|
+
}
|
|
14361
|
+
const comment = {
|
|
14362
|
+
kind: "visibility",
|
|
14363
|
+
projectId,
|
|
14364
|
+
statistics: index.statistics
|
|
14365
|
+
};
|
|
14366
|
+
await connection.unsafe(`COMMENT ON INDEX ${identifier(schema)}.${identifier(index.name)} IS ${literal(JSON.stringify(comment))}`);
|
|
14367
|
+
return;
|
|
14368
|
+
}
|
|
14369
|
+
var FUNCTION_NOT_IMMUTABLE = "42P17";
|
|
14370
|
+
async function ensureVisibilityStatistics(connection, schema, indexes) {
|
|
14371
|
+
const partitions = new Map;
|
|
14372
|
+
for (const index of indexes) {
|
|
14373
|
+
partitions.set(index.partition, [...partitions.get(index.partition) ?? [], index]);
|
|
14374
|
+
}
|
|
14375
|
+
for (const [partition, partitionIndexes] of partitions) {
|
|
14376
|
+
const table = `${identifier(schema)}.${identifier(partition)}`;
|
|
14377
|
+
await runPgTransactionOn(connection, async () => {
|
|
14378
|
+
let created = false;
|
|
14379
|
+
for (const index of partitionIndexes) {
|
|
14380
|
+
const [existing] = await connection`
|
|
14381
|
+
SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
|
|
14382
|
+
WHERE n.nspname = ${schema} AND s.stxname = ${index.statistics}`;
|
|
14383
|
+
if (existing)
|
|
14384
|
+
continue;
|
|
14385
|
+
await connection.unsafe(`CREATE STATISTICS ${identifier(schema)}.${identifier(index.statistics)} ON (${index.expression}) FROM ${table}`);
|
|
14386
|
+
created = true;
|
|
14387
|
+
}
|
|
14388
|
+
if (created)
|
|
14389
|
+
await connection.unsafe(`ANALYZE ${table}`);
|
|
14390
|
+
});
|
|
14391
|
+
}
|
|
14392
|
+
}
|
|
14393
|
+
async function dropUnusedVisibilityIndexes(connection, schema, projectId, current) {
|
|
14394
|
+
const currentNames = new Set(current.map((index) => index.name));
|
|
14395
|
+
const existing = await readVisibilityIndexes(connection, schema);
|
|
14396
|
+
const unused = existing.filter((index) => index.comment.projectId === projectId && !currentNames.has(index.name));
|
|
14397
|
+
for (const index of unused) {
|
|
14398
|
+
await connection.unsafe(`DROP INDEX CONCURRENTLY IF EXISTS ${identifier(schema)}.${identifier(index.name)}`);
|
|
14399
|
+
}
|
|
14400
|
+
const kept = existing.filter((index) => !unused.includes(index));
|
|
14401
|
+
for (const statistics of new Set(unused.map((index) => index.comment.statistics))) {
|
|
14402
|
+
if (kept.some((index) => index.comment.statistics === statistics))
|
|
14403
|
+
continue;
|
|
14404
|
+
await connection.unsafe(`DROP STATISTICS IF EXISTS ${identifier(schema)}.${identifier(statistics)}`);
|
|
14405
|
+
}
|
|
14406
|
+
}
|
|
14407
|
+
async function readVisibilityIndexes(connection, schema) {
|
|
14408
|
+
const rows = await connection`
|
|
14409
|
+
SELECT c.relname AS name, d.description AS definition
|
|
14410
|
+
FROM pg_index i
|
|
14411
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
14412
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
14413
|
+
JOIN pg_description d
|
|
14414
|
+
ON d.objoid = c.oid AND d.classoid = 'pg_class'::regclass AND d.objsubid = 0
|
|
14415
|
+
WHERE n.nspname = ${schema} AND c.relname LIKE 'sixb_visibility_%'`;
|
|
14416
|
+
return rows.flatMap((row) => {
|
|
14417
|
+
const comment = parseComment(row.definition);
|
|
14418
|
+
return comment ? [{ name: row.name, comment }] : [];
|
|
14419
|
+
});
|
|
14420
|
+
}
|
|
14421
|
+
function parseComment(definition) {
|
|
14422
|
+
try {
|
|
14423
|
+
const comment = JSON.parse(definition);
|
|
14424
|
+
return comment?.kind === "visibility" && typeof comment.projectId === "string" && typeof comment.statistics === "string" ? comment : undefined;
|
|
14425
|
+
} catch {
|
|
14426
|
+
return;
|
|
14427
|
+
}
|
|
14428
|
+
}
|
|
14429
|
+
function conditionNotIndexed(objectTypeId, error) {
|
|
14430
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
14431
|
+
return `${objectTypeId}: a view condition cannot be indexed (${reason}); members read the type by checking each object.`;
|
|
14432
|
+
}
|
|
14433
|
+
function digest(value, length) {
|
|
14434
|
+
return createHash6("sha256").update(value).digest("hex").slice(0, length);
|
|
14435
|
+
}
|
|
14436
|
+
|
|
14437
|
+
// src/objects/visibility-statistics.ts
|
|
14438
|
+
var VISIBILITY_STATISTICS_TTL_MS = 60000;
|
|
14439
|
+
|
|
14440
|
+
class PgVisibilityStatisticsCatalog {
|
|
14441
|
+
#read;
|
|
14442
|
+
commonKeys(sql) {
|
|
14443
|
+
const now = Date.now();
|
|
14444
|
+
if (this.#read && now - this.#read.at < VISIBILITY_STATISTICS_TTL_MS)
|
|
14445
|
+
return this.#read.keys;
|
|
14446
|
+
const read = { at: now, keys: readCommonKeys(sql) };
|
|
14447
|
+
this.#read = read;
|
|
14448
|
+
read.keys.catch(() => {
|
|
14449
|
+
if (this.#read === read)
|
|
14450
|
+
this.#read = undefined;
|
|
14451
|
+
});
|
|
14452
|
+
return read.keys;
|
|
14453
|
+
}
|
|
14454
|
+
}
|
|
14455
|
+
async function readCommonKeys(sql) {
|
|
14456
|
+
const rows = await sql`
|
|
14457
|
+
SELECT statistics_name AS name, most_common_elems::text::text[] AS keys
|
|
14458
|
+
FROM pg_stats_ext_exprs
|
|
14459
|
+
WHERE schemaname = current_schema() AND statistics_name LIKE 'sixb\\_visibility\\_stats\\_%'`;
|
|
14460
|
+
return new Map(rows.map((row) => [row.name, new Set(row.keys ?? [])]));
|
|
14461
|
+
}
|
|
14462
|
+
function readsFewObjects(condition, common) {
|
|
14463
|
+
const visibility = condition.visibility;
|
|
14464
|
+
if (!visibility || visibility.remainder)
|
|
14465
|
+
return false;
|
|
14466
|
+
if ((visibility.user?.id.length ?? 0) > MAX_KEYED_ID_LENGTH)
|
|
14467
|
+
return false;
|
|
14468
|
+
const readerKeys = pgVisibilityReaderKeys(visibility.user);
|
|
14469
|
+
for (const keys of visibility.keys) {
|
|
14470
|
+
const listed = common.get(visibilityStatisticsName(condition.objectTypeId, keys));
|
|
14471
|
+
if (!listed || readerKeys.some((key) => listed.has(key)))
|
|
14472
|
+
return false;
|
|
14473
|
+
}
|
|
14474
|
+
return true;
|
|
14475
|
+
}
|
|
14476
|
+
|
|
13595
14477
|
// src/objects/conditional-reader.ts
|
|
13596
14478
|
function createPgConditionalReader(sql, params, options) {
|
|
13597
14479
|
const source = compilePgConditionalObjectReadSource(params.projectId, params.scope);
|
|
13598
|
-
const
|
|
14480
|
+
const fewObjects = async (client, objectTypeId) => {
|
|
14481
|
+
const condition = params.scope.conditions.find((item) => item.objectTypeId === objectTypeId);
|
|
14482
|
+
if (!condition)
|
|
14483
|
+
return false;
|
|
14484
|
+
return readsFewObjects(condition, await options.visibilityStatistics.commonKeys(client));
|
|
14485
|
+
};
|
|
14486
|
+
const reader = new PgObjectReader(sql, source, options.aggregateAdmission, options.sortIndexes, fewObjects);
|
|
13599
14487
|
return createPgScopedReader({
|
|
13600
14488
|
projectId: params.projectId,
|
|
13601
|
-
read: (run, query) => query && hasVectorProfile3(query) ? runPgRepeatableReadTransaction(sql, (tx) => run(new PgObjectReader(tx, source))) : run(reader)
|
|
14489
|
+
read: (run, query) => query && hasVectorProfile3(query) ? runPgRepeatableReadTransaction(sql, (tx) => run(new PgObjectReader(tx, source, undefined, options.sortIndexes, fewObjects))) : run(reader)
|
|
13602
14490
|
}, options.queryCapabilities);
|
|
13603
14491
|
}
|
|
13604
14492
|
|
|
@@ -13680,7 +14568,8 @@ var PG_OBJECT_QUERY_CAPABILITIES = {
|
|
|
13680
14568
|
decimal: { equality: true, ordering: true },
|
|
13681
14569
|
date: { equality: true, ordering: true },
|
|
13682
14570
|
timestamp: { equality: true, ordering: true },
|
|
13683
|
-
userRef: { equality: true }
|
|
14571
|
+
userRef: { equality: true },
|
|
14572
|
+
artifactRef: { equality: true }
|
|
13684
14573
|
},
|
|
13685
14574
|
limits: {
|
|
13686
14575
|
totalCount: true,
|
|
@@ -13699,6 +14588,7 @@ class PgObjectStorage {
|
|
|
13699
14588
|
sortIndexes;
|
|
13700
14589
|
aggregateAdmission;
|
|
13701
14590
|
reader;
|
|
14591
|
+
visibilityStatistics = new PgVisibilityStatisticsCatalog;
|
|
13702
14592
|
constructor(sql, sortIndexes = new PgSortIndexCatalog, maxConcurrentAggregates = 1) {
|
|
13703
14593
|
this.sql = sql;
|
|
13704
14594
|
this.sortIndexes = sortIndexes;
|
|
@@ -13714,6 +14604,8 @@ class PgObjectStorage {
|
|
|
13714
14604
|
createConditionalReadScope(params) {
|
|
13715
14605
|
return createPgConditionalReader(this.sql, params, {
|
|
13716
14606
|
aggregateAdmission: this.aggregateAdmission,
|
|
14607
|
+
sortIndexes: this.sortIndexes,
|
|
14608
|
+
visibilityStatistics: this.visibilityStatistics,
|
|
13717
14609
|
queryCapabilities: () => this.queryCapabilities()
|
|
13718
14610
|
});
|
|
13719
14611
|
}
|
|
@@ -13790,6 +14682,21 @@ class PgObjectStorage {
|
|
|
13790
14682
|
...hasMore && last ? { nextPrimaryId: last.primaryId } : {}
|
|
13791
14683
|
};
|
|
13792
14684
|
}
|
|
14685
|
+
async listArtifactHolders(params) {
|
|
14686
|
+
assertArtifactHolderPageLimit(params.limit);
|
|
14687
|
+
const { after } = params;
|
|
14688
|
+
const rows = await this.sql`
|
|
14689
|
+
SELECT object_type_id, primary_id, property_id FROM object_artifact_refs
|
|
14690
|
+
WHERE project_id = ${params.projectId} AND artifact_id = ${params.artifactId}
|
|
14691
|
+
${after ? this.sql`AND (object_type_id, primary_id, property_id)
|
|
14692
|
+
> (${after.objectTypeId}, ${after.primaryId}, ${after.propertyId})` : this.sql``}
|
|
14693
|
+
ORDER BY object_type_id, primary_id, property_id
|
|
14694
|
+
LIMIT ${params.limit + 1}
|
|
14695
|
+
`;
|
|
14696
|
+
const holders = rows.slice(0, params.limit).map(artifactHolderFromRow);
|
|
14697
|
+
const last = holders.at(-1);
|
|
14698
|
+
return { holders, ...rows.length > params.limit && last ? { next: last } : {} };
|
|
14699
|
+
}
|
|
13793
14700
|
async list(params) {
|
|
13794
14701
|
return this.reader.list(params);
|
|
13795
14702
|
}
|
|
@@ -13815,8 +14722,20 @@ function assertReconciliationPageLimit(limit) {
|
|
|
13815
14722
|
throw new Error("Object reconciliation page limit must be a positive safe integer.");
|
|
13816
14723
|
}
|
|
13817
14724
|
}
|
|
14725
|
+
function artifactHolderFromRow(row) {
|
|
14726
|
+
return {
|
|
14727
|
+
objectTypeId: row.object_type_id,
|
|
14728
|
+
primaryId: row.primary_id,
|
|
14729
|
+
propertyId: row.property_id
|
|
14730
|
+
};
|
|
14731
|
+
}
|
|
14732
|
+
function assertArtifactHolderPageLimit(limit) {
|
|
14733
|
+
if (!Number.isSafeInteger(limit) || limit <= 0) {
|
|
14734
|
+
throw new Error("Artifact holder page limit must be a positive safe integer.");
|
|
14735
|
+
}
|
|
14736
|
+
}
|
|
13818
14737
|
// src/objects/prepare-queries.ts
|
|
13819
|
-
import { createHash as
|
|
14738
|
+
import { createHash as createHash7 } from "crypto";
|
|
13820
14739
|
async function preparePgObjectQueries(sql, schema, plan) {
|
|
13821
14740
|
const fingerprint = preparationFingerprint(plan);
|
|
13822
14741
|
const current = await readCompletedPreparation(sql, fingerprint);
|
|
@@ -13849,10 +14768,11 @@ function preparationFingerprint(plan) {
|
|
|
13849
14768
|
objectTypeId: type.objectTypeId,
|
|
13850
14769
|
properties: [...type.properties].sort((a, b) => a.propertyId.localeCompare(b.propertyId)),
|
|
13851
14770
|
indexes: type.indexes.map(normalizeIndex).sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
|
|
13852
|
-
links: [...type.links].sort((a, b) => a.linkId.localeCompare(b.linkId))
|
|
14771
|
+
links: [...type.links].sort((a, b) => a.linkId.localeCompare(b.linkId)),
|
|
14772
|
+
...type.visibility ? { visibility: type.visibility.map(compilePgVisibilityKeys) } : {}
|
|
13853
14773
|
})).sort((a, b) => a.objectTypeId.localeCompare(b.objectTypeId));
|
|
13854
|
-
const definition = { version:
|
|
13855
|
-
return
|
|
14774
|
+
const definition = { version: 5, projectId: plan.projectId, objectTypes };
|
|
14775
|
+
return createHash7("sha256").update(JSON.stringify(definition)).digest("hex");
|
|
13856
14776
|
}
|
|
13857
14777
|
function normalizeIndex(index) {
|
|
13858
14778
|
if (index.kind === "sort") {
|
|
@@ -13889,11 +14809,12 @@ async function prepareStructures(sql, schema, plan) {
|
|
|
13889
14809
|
await ensureObjectPartition(sql, schema, type.objectTypeId);
|
|
13890
14810
|
const names = await ensurePgObjectQueryIndexes(sql, schema, indexes);
|
|
13891
14811
|
const texts = await preparePgObjectTextCounts(sql, schema, textCounts);
|
|
14812
|
+
const visibility = await preparePgVisibilityIndexes(sql, schema, plan);
|
|
13892
14813
|
return {
|
|
13893
14814
|
status: "prepared",
|
|
13894
14815
|
objectTypes: plan.objectTypes.length,
|
|
13895
|
-
indexes: new Set([...names, ...texts]).size,
|
|
13896
|
-
warnings
|
|
14816
|
+
indexes: new Set([...names, ...texts, ...visibility.names]).size,
|
|
14817
|
+
warnings: [...warnings, ...visibility.warnings]
|
|
13897
14818
|
};
|
|
13898
14819
|
}
|
|
13899
14820
|
async function linkWarnings(sql, plan) {
|
|
@@ -15938,6 +16859,7 @@ class PgMaterializationSessionState extends ProviderMaterializationSessionState
|
|
|
15938
16859
|
changedLinks = 0;
|
|
15939
16860
|
changedUniqueValues = 0;
|
|
15940
16861
|
uniqueWork = { releases: false, claims: false };
|
|
16862
|
+
artifactHolderWork = { deletes: false, inserts: false };
|
|
15941
16863
|
workAnalyzed = false;
|
|
15942
16864
|
workTable;
|
|
15943
16865
|
workId;
|
|
@@ -16041,6 +16963,10 @@ class PgMaterializationSessions {
|
|
|
16041
16963
|
session.uniqueWork.releases = true;
|
|
16042
16964
|
if (record.item.kind === "unique-value-claim")
|
|
16043
16965
|
session.uniqueWork.claims = true;
|
|
16966
|
+
if (record.item.kind === "artifact-holder-delete")
|
|
16967
|
+
session.artifactHolderWork.deletes = true;
|
|
16968
|
+
if (record.item.kind === "artifact-holder-insert")
|
|
16969
|
+
session.artifactHolderWork.inserts = true;
|
|
16044
16970
|
}
|
|
16045
16971
|
}
|
|
16046
16972
|
async analyzeSealedWork(session) {
|
|
@@ -16335,13 +17261,18 @@ class PgMaterializationWriter {
|
|
|
16335
17261
|
await this.applyLinkOverrides(target);
|
|
16336
17262
|
await this.applyPoints(target);
|
|
16337
17263
|
const uniqueWork = target.uniqueWork ?? await this.stagedUniqueWork(target);
|
|
17264
|
+
const holderWork = target.artifactHolderWork ?? await this.stagedArtifactHolderWork(target);
|
|
16338
17265
|
if (uniqueWork.releases)
|
|
16339
17266
|
await this.releaseUniqueValues(target);
|
|
17267
|
+
if (holderWork.deletes)
|
|
17268
|
+
await this.deleteArtifactHolders(target);
|
|
16340
17269
|
let linkWrites = await this.deleteLinks(target);
|
|
16341
17270
|
let objectWrites = await this.deleteObjects(target);
|
|
16342
17271
|
objectWrites += await this.upsertObjects(target);
|
|
16343
17272
|
linkWrites += await this.upsertLinks(target);
|
|
16344
17273
|
const uniqueValueWrites = uniqueWork.claims ? await this.claimUniqueValues(target) : 0;
|
|
17274
|
+
if (holderWork.inserts)
|
|
17275
|
+
await this.insertArtifactHolders(target);
|
|
16345
17276
|
const eventCount = await this.writeOutbox(target);
|
|
16346
17277
|
await this.writeTouches(target);
|
|
16347
17278
|
return { objectWrites, linkWrites, uniqueValueWrites, eventCount };
|
|
@@ -16372,6 +17303,21 @@ class PgMaterializationWriter {
|
|
|
16372
17303
|
`;
|
|
16373
17304
|
return { releases: row?.releases ?? false, claims: row?.claims ?? false };
|
|
16374
17305
|
}
|
|
17306
|
+
async stagedArtifactHolderWork(target) {
|
|
17307
|
+
const probe = (kind) => this.sql`
|
|
17308
|
+
EXISTS (
|
|
17309
|
+
SELECT 1 FROM ${this.sql(target.workTable)}
|
|
17310
|
+
WHERE work_id = ${target.workId} AND lane = 'apply'
|
|
17311
|
+
AND major_order = ${materializationApplyPhase2(kind)}
|
|
17312
|
+
AND minor_order = ${materializationPlanKindRank2(kind)}
|
|
17313
|
+
)
|
|
17314
|
+
`;
|
|
17315
|
+
const [row] = await this.sql`
|
|
17316
|
+
SELECT ${probe("artifact-holder-delete")} AS deletes,
|
|
17317
|
+
${probe("artifact-holder-insert")} AS inserts
|
|
17318
|
+
`;
|
|
17319
|
+
return { deletes: row?.deletes ?? false, inserts: row?.inserts ?? false };
|
|
17320
|
+
}
|
|
16375
17321
|
staged(target, kind) {
|
|
16376
17322
|
return this.sql`
|
|
16377
17323
|
SELECT payload->'item'->'value' AS value
|
|
@@ -16634,6 +17580,34 @@ class PgMaterializationWriter {
|
|
|
16634
17580
|
async claimUniqueValues(target) {
|
|
16635
17581
|
return claimPgUniqueValues(this.sql, target.projectId, this.stagedUniqueValues(target, "unique-value-claim"), { requirePrepared: true });
|
|
16636
17582
|
}
|
|
17583
|
+
stagedArtifactHolders(target, kind) {
|
|
17584
|
+
return this.sql`
|
|
17585
|
+
SELECT value->>'artifactId' AS artifact_id, value->>'objectTypeId' AS object_type_id,
|
|
17586
|
+
value->>'primaryId' AS primary_id, value->>'propertyId' AS property_id
|
|
17587
|
+
FROM (${this.staged(target, kind)}) AS staged
|
|
17588
|
+
`;
|
|
17589
|
+
}
|
|
17590
|
+
async deleteArtifactHolders(target) {
|
|
17591
|
+
await this.sql`
|
|
17592
|
+
WITH candidates AS (${this.stagedArtifactHolders(target, "artifact-holder-delete")})
|
|
17593
|
+
DELETE FROM object_artifact_refs AS stored USING candidates
|
|
17594
|
+
WHERE stored.project_id = ${target.projectId}
|
|
17595
|
+
AND stored.artifact_id = candidates.artifact_id
|
|
17596
|
+
AND stored.object_type_id = candidates.object_type_id
|
|
17597
|
+
AND stored.primary_id = candidates.primary_id
|
|
17598
|
+
AND stored.property_id = candidates.property_id
|
|
17599
|
+
`;
|
|
17600
|
+
}
|
|
17601
|
+
async insertArtifactHolders(target) {
|
|
17602
|
+
await this.sql`
|
|
17603
|
+
INSERT INTO object_artifact_refs (
|
|
17604
|
+
project_id, artifact_id, object_type_id, primary_id, property_id
|
|
17605
|
+
)
|
|
17606
|
+
SELECT ${target.projectId}, artifact_id, object_type_id, primary_id, property_id
|
|
17607
|
+
FROM (${this.stagedArtifactHolders(target, "artifact-holder-insert")}) AS candidates
|
|
17608
|
+
ON CONFLICT DO NOTHING
|
|
17609
|
+
`;
|
|
17610
|
+
}
|
|
16637
17611
|
async deleteLinks(target) {
|
|
16638
17612
|
const candidates = this.staged(target, "link-delete");
|
|
16639
17613
|
return this.write(this.sql`
|
|
@@ -16830,7 +17804,8 @@ class PgMaterializationWriter {
|
|
|
16830
17804
|
END AS ref
|
|
16831
17805
|
) AS entity
|
|
16832
17806
|
WHERE work.work_id = ${target.workId} AND work.lane = 'apply'
|
|
16833
|
-
-- Unique values are no plan input: a stale claim fails when it applies.
|
|
17807
|
+
-- Unique values are no plan input: a stale claim fails when it applies. Nor are the
|
|
17808
|
+
-- artifacts objects hold, which follow the objects.
|
|
16834
17809
|
AND entity.kind IS NOT NULL
|
|
16835
17810
|
`;
|
|
16836
17811
|
}
|
|
@@ -17028,7 +18003,7 @@ class PgOntologyMaterializationStorage {
|
|
|
17028
18003
|
projectId: commit.projectId,
|
|
17029
18004
|
commitId: commit.id,
|
|
17030
18005
|
committedAt: commit.committedAt,
|
|
17031
|
-
...session.plan ? {} : { uniqueWork: session.uniqueWork }
|
|
18006
|
+
...session.plan ? {} : { uniqueWork: session.uniqueWork, artifactHolderWork: session.artifactHolderWork }
|
|
17032
18007
|
});
|
|
17033
18008
|
if (applied.objectWrites >= STATISTICS_MINIMUM_CHANGES) {
|
|
17034
18009
|
session.changedObjects = await this.writer.stagedObjectWritesByType({
|
|
@@ -20997,30 +21972,62 @@ function invalidState(reservation, operation) {
|
|
|
20997
21972
|
}
|
|
20998
21973
|
|
|
20999
21974
|
// src/pg-artifact-storage.ts
|
|
21975
|
+
import { ARTIFACT_COMMENT_LIMITS } from "@sixb/core/artifacts";
|
|
21000
21976
|
import {
|
|
21977
|
+
ARTIFACT_RESTORE_POINTS,
|
|
21001
21978
|
artifactStateBytes,
|
|
21002
21979
|
artifactStorageErrors,
|
|
21003
21980
|
checkArtifactStateWrite,
|
|
21981
|
+
othersCommented,
|
|
21982
|
+
parseArtifactBoardPreviewRow,
|
|
21983
|
+
parseArtifactCommentRow,
|
|
21004
21984
|
parseArtifactFrameGrantRow,
|
|
21985
|
+
parseArtifactRestorePointRow,
|
|
21005
21986
|
parseArtifactRow,
|
|
21006
21987
|
parseArtifactStateRow,
|
|
21007
|
-
|
|
21988
|
+
parseArtifactTurnWriteRow,
|
|
21989
|
+
parseArtifactVersionRow,
|
|
21990
|
+
turnUndoable
|
|
21008
21991
|
} from "@sixb/core/internal/storage";
|
|
21009
21992
|
var ARTIFACT_COLUMNS = `
|
|
21010
21993
|
project_id AS "projectId", id, kind, title,
|
|
21011
21994
|
owner_principal_type AS "ownerPrincipalType", owner_principal_id AS "ownerPrincipalId",
|
|
21012
|
-
latest_version AS "latestVersion", created_at AS "createdAt",
|
|
21995
|
+
latest_version AS "latestVersion", revision, created_at AS "createdAt",
|
|
21996
|
+
updated_at AS "updatedAt", document, document_revision AS "documentRevision"
|
|
21997
|
+
`;
|
|
21998
|
+
var CONTENT_COLUMNS = `
|
|
21999
|
+
title, entry, files, outline, refs, author_principal_type AS "authorPrincipalType",
|
|
22000
|
+
author_principal_id AS "authorPrincipalId", origin_thread_id AS "originThreadId",
|
|
22001
|
+
origin_run_id AS "originRunId", updated_at AS "updatedAt"
|
|
21013
22002
|
`;
|
|
21014
22003
|
var VERSION_COLUMNS = `
|
|
21015
|
-
project_id AS "projectId", artifact_id AS "artifactId", version,
|
|
21016
|
-
|
|
21017
|
-
|
|
22004
|
+
project_id AS "projectId", artifact_id AS "artifactId", version, revision, ${CONTENT_COLUMNS},
|
|
22005
|
+
created_at AS "createdAt", frozen_at AS "frozenAt",
|
|
22006
|
+
frozen_by_principal_type AS "frozenByPrincipalType",
|
|
22007
|
+
frozen_by_principal_id AS "frozenByPrincipalId"
|
|
21018
22008
|
`;
|
|
22009
|
+
var RESTORE_POINT_COLUMNS = `
|
|
22010
|
+
project_id AS "projectId", artifact_id AS "artifactId", revision, version, frozen,
|
|
22011
|
+
${CONTENT_COLUMNS}, reason, turn_run_id AS "turnRunId", created_at AS "createdAt"
|
|
22012
|
+
`;
|
|
22013
|
+
var TURN_WRITE_COLUMNS = `artifact_id AS "artifactId", created, revision, writes`;
|
|
21019
22014
|
var FRAME_GRANT_COLUMNS = `
|
|
21020
22015
|
project_id AS "projectId", token_hash AS "tokenHash", artifact_id AS "artifactId", version,
|
|
21021
|
-
targets, created_at AS "createdAt", expires_at AS "expiresAt"
|
|
22016
|
+
files_digest AS "filesDigest", targets, created_at AS "createdAt", expires_at AS "expiresAt"
|
|
22017
|
+
`;
|
|
22018
|
+
var COMMENT_COLUMNS = `
|
|
22019
|
+
project_id AS "projectId", artifact_id AS "artifactId", id, parent_id AS "parentId", anchor,
|
|
22020
|
+
body, author_principal_type AS "authorPrincipalType", author_principal_id AS "authorPrincipalId",
|
|
22021
|
+
author_name AS "authorName", created_at AS "createdAt", resolved_at AS "resolvedAt",
|
|
22022
|
+
resolved_by_principal_type AS "resolvedByPrincipalType",
|
|
22023
|
+
resolved_by_principal_id AS "resolvedByPrincipalId",
|
|
22024
|
+
origin_thread_id AS "originThreadId", origin_run_id AS "originRunId"
|
|
21022
22025
|
`;
|
|
21023
22026
|
var STATE_COLUMNS = `key, value, size_bytes AS "sizeBytes", updated_at AS "updatedAt"`;
|
|
22027
|
+
var BOARD_PREVIEW_COLUMNS = `
|
|
22028
|
+
project_id AS "projectId", render_key AS "renderKey", theme, size, file,
|
|
22029
|
+
created_at AS "createdAt"
|
|
22030
|
+
`;
|
|
21024
22031
|
|
|
21025
22032
|
class PgArtifactStorage {
|
|
21026
22033
|
sql;
|
|
@@ -21029,61 +22036,223 @@ class PgArtifactStorage {
|
|
|
21029
22036
|
}
|
|
21030
22037
|
async create(input) {
|
|
21031
22038
|
return runPgTransaction(this.sql, async (tx) => {
|
|
22039
|
+
const content = input.version;
|
|
21032
22040
|
const [artifact] = await tx`
|
|
21033
22041
|
INSERT INTO artifacts (
|
|
21034
22042
|
project_id, id, kind, title, owner_principal_type, owner_principal_id,
|
|
21035
|
-
latest_version, next_version, created_at, updated_at
|
|
22043
|
+
latest_version, next_version, revision, created_at, updated_at, document,
|
|
22044
|
+
document_revision
|
|
21036
22045
|
) VALUES (
|
|
21037
|
-
${input.projectId}, ${input.id}, ${input.kind}, ${
|
|
21038
|
-
${input.owner.type}, ${input.owner.id}, 1, 2,
|
|
21039
|
-
${
|
|
22046
|
+
${input.projectId}, ${input.id}, ${input.kind}, ${content.title},
|
|
22047
|
+
${input.owner.type}, ${input.owner.id}, 1, 2, 1,
|
|
22048
|
+
${content.updatedAt}, ${content.updatedAt}, ${documentJson(input.document)}::text::jsonb,
|
|
22049
|
+
${input.document ? 1 : null}
|
|
21040
22050
|
)
|
|
21041
22051
|
ON CONFLICT (project_id, id) DO NOTHING
|
|
21042
22052
|
RETURNING ${tx.unsafe(ARTIFACT_COLUMNS)}
|
|
21043
22053
|
`;
|
|
21044
22054
|
if (!artifact)
|
|
21045
22055
|
throw artifactStorageErrors.exists(input.id);
|
|
21046
|
-
|
|
21047
|
-
|
|
21048
|
-
|
|
21049
|
-
|
|
22056
|
+
const version = await insertVersion(tx, {
|
|
22057
|
+
projectId: input.projectId,
|
|
22058
|
+
artifactId: input.id,
|
|
22059
|
+
version: 1,
|
|
22060
|
+
revision: 1,
|
|
22061
|
+
content
|
|
22062
|
+
});
|
|
22063
|
+
await shareIn(tx, input.projectId, input.id, content, input.turnRunId);
|
|
22064
|
+
if (input.turnRunId) {
|
|
22065
|
+
await recordTurnWrite(tx, {
|
|
22066
|
+
projectId: input.projectId,
|
|
22067
|
+
artifactId: input.id,
|
|
22068
|
+
turnRunId: input.turnRunId,
|
|
22069
|
+
created: true,
|
|
22070
|
+
revision: 1
|
|
22071
|
+
});
|
|
22072
|
+
}
|
|
22073
|
+
return { artifact: parseArtifactRow(artifact), version };
|
|
21050
22074
|
});
|
|
21051
22075
|
}
|
|
21052
22076
|
async publish(input) {
|
|
21053
22077
|
return runPgTransaction(this.sql, async (tx) => {
|
|
21054
|
-
const
|
|
21055
|
-
SELECT kind, next_version, latest_version FROM artifacts
|
|
21056
|
-
WHERE project_id = ${input.projectId} AND id = ${input.artifactId}
|
|
21057
|
-
FOR UPDATE
|
|
21058
|
-
`;
|
|
22078
|
+
const current = await lockArtifact(tx, input.projectId, input.artifactId);
|
|
21059
22079
|
if (!current)
|
|
21060
22080
|
throw artifactStorageErrors.notFound(input.artifactId);
|
|
21061
|
-
const
|
|
21062
|
-
if (kind !== input.kind) {
|
|
21063
|
-
throw artifactStorageErrors.kindMismatch(input.artifactId, kind, input.kind);
|
|
22081
|
+
const { artifact } = current;
|
|
22082
|
+
if (artifact.kind !== input.kind) {
|
|
22083
|
+
throw artifactStorageErrors.kindMismatch(input.artifactId, artifact.kind, input.kind);
|
|
21064
22084
|
}
|
|
21065
|
-
const
|
|
21066
|
-
|
|
21067
|
-
|
|
21068
|
-
|
|
21069
|
-
|
|
21070
|
-
|
|
21071
|
-
|
|
21072
|
-
|
|
21073
|
-
throw artifactStorageErrors.moved(input.artifactId, input.baseVersion, {
|
|
21074
|
-
version: latest.version,
|
|
21075
|
-
author: latest.author
|
|
22085
|
+
const latest = await latestVersion(tx, artifact);
|
|
22086
|
+
assertBase(artifact, latest, input.baseRevision);
|
|
22087
|
+
const { turnRunId } = input;
|
|
22088
|
+
if (turnRunId && !await turnWrote(tx, artifact, turnRunId)) {
|
|
22089
|
+
await keepRestorePoint(tx, latest, {
|
|
22090
|
+
reason: "turn",
|
|
22091
|
+
turnRunId,
|
|
22092
|
+
createdAt: input.version.updatedAt
|
|
21076
22093
|
});
|
|
21077
22094
|
}
|
|
21078
|
-
const
|
|
21079
|
-
|
|
22095
|
+
const written = await writeLatest(tx, current, latest, input.version, input.document);
|
|
22096
|
+
await shareIn(tx, input.projectId, input.artifactId, input.version, turnRunId);
|
|
22097
|
+
if (turnRunId) {
|
|
22098
|
+
await recordTurnWrite(tx, {
|
|
22099
|
+
projectId: input.projectId,
|
|
22100
|
+
artifactId: input.artifactId,
|
|
22101
|
+
turnRunId,
|
|
22102
|
+
created: false,
|
|
22103
|
+
revision: written.artifact.revision
|
|
22104
|
+
});
|
|
22105
|
+
}
|
|
22106
|
+
return written;
|
|
22107
|
+
});
|
|
22108
|
+
}
|
|
22109
|
+
async freeze(input) {
|
|
22110
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
22111
|
+
const current = await lockArtifact(tx, input.projectId, input.artifactId);
|
|
22112
|
+
if (!current)
|
|
22113
|
+
throw artifactStorageErrors.notFound(input.artifactId);
|
|
22114
|
+
const { artifact } = current;
|
|
22115
|
+
const latest = await latestVersion(tx, artifact);
|
|
22116
|
+
if (latest.frozenAt)
|
|
22117
|
+
return { artifact, version: latest };
|
|
22118
|
+
assertBase(artifact, latest, input.baseRevision);
|
|
22119
|
+
const revision = artifact.revision + 1;
|
|
22120
|
+
const [version] = await tx`
|
|
22121
|
+
UPDATE artifact_versions
|
|
22122
|
+
SET revision = ${revision}, frozen_at = ${input.frozenAt},
|
|
22123
|
+
frozen_by_principal_type = ${input.frozenBy.type},
|
|
22124
|
+
frozen_by_principal_id = ${input.frozenBy.id}
|
|
22125
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22126
|
+
AND version = ${latest.version}
|
|
22127
|
+
RETURNING ${tx.unsafe(VERSION_COLUMNS)}
|
|
22128
|
+
`;
|
|
22129
|
+
const [updated] = await tx`
|
|
21080
22130
|
UPDATE artifacts
|
|
21081
|
-
SET
|
|
21082
|
-
|
|
22131
|
+
SET revision = ${revision}, updated_at = ${input.frozenAt},
|
|
22132
|
+
-- A freeze keeps the content, so a document that held it still does.
|
|
22133
|
+
document_revision = CASE WHEN document_revision = ${artifact.revision}::int
|
|
22134
|
+
THEN ${revision}::int ELSE document_revision END
|
|
21083
22135
|
WHERE project_id = ${input.projectId} AND id = ${input.artifactId}
|
|
21084
22136
|
RETURNING ${tx.unsafe(ARTIFACT_COLUMNS)}
|
|
21085
22137
|
`;
|
|
21086
|
-
return {
|
|
22138
|
+
return {
|
|
22139
|
+
artifact: parseArtifactRow(updated),
|
|
22140
|
+
version: parseArtifactVersionRow(version)
|
|
22141
|
+
};
|
|
22142
|
+
});
|
|
22143
|
+
}
|
|
22144
|
+
async listRestorePoints(input) {
|
|
22145
|
+
const rows = await this.sql`
|
|
22146
|
+
SELECT ${this.sql.unsafe(RESTORE_POINT_COLUMNS)} FROM artifact_restore_points
|
|
22147
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.id}
|
|
22148
|
+
ORDER BY revision DESC
|
|
22149
|
+
`;
|
|
22150
|
+
return rows.map(parseArtifactRestorePointRow);
|
|
22151
|
+
}
|
|
22152
|
+
async getRestorePoint(input) {
|
|
22153
|
+
const [row] = await this.sql`
|
|
22154
|
+
SELECT ${this.sql.unsafe(RESTORE_POINT_COLUMNS)} FROM artifact_restore_points
|
|
22155
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22156
|
+
AND revision = ${input.revision}
|
|
22157
|
+
`;
|
|
22158
|
+
return row ? parseArtifactRestorePointRow(row) : null;
|
|
22159
|
+
}
|
|
22160
|
+
async restoreTo(input) {
|
|
22161
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
22162
|
+
const current = await lockArtifact(tx, input.projectId, input.artifactId);
|
|
22163
|
+
if (!current)
|
|
22164
|
+
throw artifactStorageErrors.notFound(input.artifactId);
|
|
22165
|
+
const { artifact } = current;
|
|
22166
|
+
const [row] = await tx`
|
|
22167
|
+
SELECT ${tx.unsafe(RESTORE_POINT_COLUMNS)} FROM artifact_restore_points
|
|
22168
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22169
|
+
AND revision = ${input.revision}
|
|
22170
|
+
`;
|
|
22171
|
+
if (!row)
|
|
22172
|
+
throw artifactStorageErrors.restorePointNotFound(input.artifactId, input.revision);
|
|
22173
|
+
const point = parseArtifactRestorePointRow(row);
|
|
22174
|
+
const latest = await latestVersion(tx, artifact);
|
|
22175
|
+
assertBase(artifact, latest, input.baseRevision);
|
|
22176
|
+
await keepRestorePoint(tx, latest, { reason: "restore", createdAt: input.restoredAt });
|
|
22177
|
+
return writeLatest(tx, current, latest, {
|
|
22178
|
+
title: point.title,
|
|
22179
|
+
entry: point.entry,
|
|
22180
|
+
files: point.files,
|
|
22181
|
+
outline: point.outline,
|
|
22182
|
+
references: point.references,
|
|
22183
|
+
author: input.author,
|
|
22184
|
+
updatedAt: input.restoredAt
|
|
22185
|
+
}, input.document);
|
|
22186
|
+
});
|
|
22187
|
+
}
|
|
22188
|
+
async listTurnWrites(input) {
|
|
22189
|
+
const rows = await this.sql`
|
|
22190
|
+
SELECT ${this.sql.unsafe(TURN_WRITE_COLUMNS)} FROM artifact_turn_writes
|
|
22191
|
+
WHERE project_id = ${input.projectId} AND turn_run_id = ${input.turnRunId}
|
|
22192
|
+
ORDER BY artifact_id COLLATE "C"
|
|
22193
|
+
`;
|
|
22194
|
+
return rows.map(parseArtifactTurnWriteRow);
|
|
22195
|
+
}
|
|
22196
|
+
async undoTurnWrites(input) {
|
|
22197
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
22198
|
+
const { projectId, turnRunId } = input;
|
|
22199
|
+
await tx`
|
|
22200
|
+
SELECT id FROM artifacts
|
|
22201
|
+
WHERE project_id = ${projectId} AND id IN (
|
|
22202
|
+
SELECT artifact_id FROM artifact_turn_writes
|
|
22203
|
+
WHERE project_id = ${projectId} AND turn_run_id = ${turnRunId}
|
|
22204
|
+
)
|
|
22205
|
+
ORDER BY id
|
|
22206
|
+
FOR UPDATE
|
|
22207
|
+
`;
|
|
22208
|
+
const rows = await tx`
|
|
22209
|
+
SELECT w.artifact_id AS "artifactId", w.created, w.revision, w.writes,
|
|
22210
|
+
a.revision AS "currentRevision"
|
|
22211
|
+
FROM artifact_turn_writes w
|
|
22212
|
+
JOIN artifacts a ON a.project_id = w.project_id AND a.id = w.artifact_id
|
|
22213
|
+
WHERE w.project_id = ${projectId} AND w.turn_run_id = ${turnRunId}
|
|
22214
|
+
ORDER BY w.artifact_id COLLATE "C"
|
|
22215
|
+
`;
|
|
22216
|
+
const restored = [];
|
|
22217
|
+
const deleted = [];
|
|
22218
|
+
for (const row of rows) {
|
|
22219
|
+
const write = parseArtifactTurnWriteRow(row);
|
|
22220
|
+
const revision = Number(row.currentRevision);
|
|
22221
|
+
if (write.created) {
|
|
22222
|
+
if (!turnUndoable({ write, revision }))
|
|
22223
|
+
continue;
|
|
22224
|
+
const comments = await tx`
|
|
22225
|
+
SELECT ${tx.unsafe(COMMENT_COLUMNS)} FROM artifact_comments
|
|
22226
|
+
WHERE project_id = ${projectId} AND artifact_id = ${write.artifactId}
|
|
22227
|
+
`;
|
|
22228
|
+
if (othersCommented(comments.map(parseArtifactCommentRow), input))
|
|
22229
|
+
continue;
|
|
22230
|
+
await tx`DELETE FROM artifacts WHERE project_id = ${projectId} AND id = ${write.artifactId}`;
|
|
22231
|
+
deleted.push(write.artifactId);
|
|
22232
|
+
continue;
|
|
22233
|
+
}
|
|
22234
|
+
const undone = await undoTurn(tx, projectId, write, turnRunId, input.documents?.[write.artifactId]);
|
|
22235
|
+
if (undone)
|
|
22236
|
+
restored.push(undone);
|
|
22237
|
+
}
|
|
22238
|
+
if (restored.length > 0) {
|
|
22239
|
+
const undone = restored.map(({ artifact }) => artifact.id);
|
|
22240
|
+
await tx`
|
|
22241
|
+
DELETE FROM artifact_restore_points
|
|
22242
|
+
WHERE project_id = ${projectId} AND turn_run_id = ${turnRunId}
|
|
22243
|
+
AND artifact_id IN ${tx(undone)}
|
|
22244
|
+
`;
|
|
22245
|
+
await tx`
|
|
22246
|
+
DELETE FROM artifact_threads
|
|
22247
|
+
WHERE project_id = ${projectId} AND linked_by_run_id = ${turnRunId} AND NOT copied
|
|
22248
|
+
AND artifact_id IN ${tx(undone)}
|
|
22249
|
+
`;
|
|
22250
|
+
}
|
|
22251
|
+
await tx`
|
|
22252
|
+
DELETE FROM artifact_turn_writes
|
|
22253
|
+
WHERE project_id = ${projectId} AND turn_run_id = ${turnRunId}
|
|
22254
|
+
`;
|
|
22255
|
+
return { restored, deleted };
|
|
21087
22256
|
});
|
|
21088
22257
|
}
|
|
21089
22258
|
async get(input) {
|
|
@@ -21150,10 +22319,12 @@ class PgArtifactStorage {
|
|
|
21150
22319
|
const [artifact] = await tx`
|
|
21151
22320
|
INSERT INTO artifacts (
|
|
21152
22321
|
project_id, id, kind, title, owner_principal_type, owner_principal_id,
|
|
21153
|
-
latest_version, next_version, created_at, updated_at
|
|
22322
|
+
latest_version, next_version, revision, created_at, updated_at, document,
|
|
22323
|
+
document_revision
|
|
21154
22324
|
)
|
|
21155
22325
|
SELECT project_id, ${input.id}, kind, title, ${input.owner.type}, ${input.owner.id},
|
|
21156
|
-
latest_version, next_version, created_at, updated_at
|
|
22326
|
+
latest_version, next_version, revision, created_at, updated_at, document,
|
|
22327
|
+
document_revision
|
|
21157
22328
|
FROM artifacts WHERE project_id = ${input.projectId} AND id = ${input.sourceId}
|
|
21158
22329
|
ON CONFLICT (project_id, id) DO NOTHING
|
|
21159
22330
|
RETURNING ${tx.unsafe(ARTIFACT_COLUMNS)}
|
|
@@ -21162,14 +22333,32 @@ class PgArtifactStorage {
|
|
|
21162
22333
|
throw artifactStorageErrors.exists(input.id);
|
|
21163
22334
|
await tx`
|
|
21164
22335
|
INSERT INTO artifact_versions (
|
|
21165
|
-
project_id, artifact_id, version, title, entry, files, outline, refs,
|
|
21166
|
-
author_principal_type, author_principal_id, created_at
|
|
22336
|
+
project_id, artifact_id, version, revision, title, entry, files, outline, refs,
|
|
22337
|
+
author_principal_type, author_principal_id, created_at, updated_at, frozen_at,
|
|
22338
|
+
frozen_by_principal_type, frozen_by_principal_id
|
|
21167
22339
|
)
|
|
21168
|
-
SELECT project_id, ${input.id}, version, title, entry, files, outline, refs,
|
|
21169
|
-
author_principal_type, author_principal_id, created_at
|
|
22340
|
+
SELECT project_id, ${input.id}, version, revision, title, entry, files, outline, refs,
|
|
22341
|
+
author_principal_type, author_principal_id, created_at, updated_at, frozen_at,
|
|
22342
|
+
frozen_by_principal_type, frozen_by_principal_id
|
|
21170
22343
|
FROM artifact_versions
|
|
21171
22344
|
WHERE project_id = ${input.projectId} AND artifact_id = ${input.sourceId}
|
|
21172
22345
|
`;
|
|
22346
|
+
if (input.outlineOf) {
|
|
22347
|
+
const copied = await tx`
|
|
22348
|
+
SELECT ${tx.unsafe(VERSION_COLUMNS)} FROM artifact_versions
|
|
22349
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.id}
|
|
22350
|
+
`;
|
|
22351
|
+
for (const version of copied.map(parseArtifactVersionRow)) {
|
|
22352
|
+
const outline = input.outlineOf(version);
|
|
22353
|
+
if (!outline)
|
|
22354
|
+
continue;
|
|
22355
|
+
await tx`
|
|
22356
|
+
UPDATE artifact_versions SET outline = ${JSON.stringify(outline)}::text::jsonb
|
|
22357
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.id}
|
|
22358
|
+
AND version = ${version.version}
|
|
22359
|
+
`;
|
|
22360
|
+
}
|
|
22361
|
+
}
|
|
21173
22362
|
await tx`
|
|
21174
22363
|
INSERT INTO artifact_threads (project_id, artifact_id, thread_id, shared_at, copied)
|
|
21175
22364
|
VALUES (
|
|
@@ -21216,10 +22405,11 @@ class PgArtifactStorage {
|
|
|
21216
22405
|
async createFrameGrant(record) {
|
|
21217
22406
|
const rows = await this.sql`
|
|
21218
22407
|
INSERT INTO artifact_frame_grants
|
|
21219
|
-
(project_id, token_hash, artifact_id, version, targets, created_at,
|
|
22408
|
+
(project_id, token_hash, artifact_id, version, files_digest, targets, created_at,
|
|
22409
|
+
expires_at)
|
|
21220
22410
|
SELECT ${record.projectId}, ${record.tokenHash}, ${record.artifactId}, ${record.version},
|
|
21221
|
-
${JSON.stringify(record.targets)}::text::jsonb,
|
|
21222
|
-
${record.expiresAt}
|
|
22411
|
+
${record.filesDigest}, ${JSON.stringify(record.targets)}::text::jsonb,
|
|
22412
|
+
${record.createdAt}, ${record.expiresAt}
|
|
21223
22413
|
WHERE EXISTS (
|
|
21224
22414
|
SELECT 1 FROM artifacts WHERE project_id = ${record.projectId} AND id = ${record.artifactId}
|
|
21225
22415
|
)
|
|
@@ -21327,91 +22517,370 @@ class PgArtifactStorage {
|
|
|
21327
22517
|
`;
|
|
21328
22518
|
return rows.length;
|
|
21329
22519
|
}
|
|
21330
|
-
async
|
|
22520
|
+
async listBoardPreviews(input) {
|
|
22521
|
+
if (input.renderKeys.length === 0)
|
|
22522
|
+
return [];
|
|
21331
22523
|
const rows = await this.sql`
|
|
21332
|
-
|
|
21333
|
-
|
|
22524
|
+
SELECT ${this.sql.unsafe(BOARD_PREVIEW_COLUMNS)} FROM artifact_board_previews
|
|
22525
|
+
WHERE project_id = ${input.projectId}
|
|
22526
|
+
AND render_key = ANY(${this.sql.array([...new Set(input.renderKeys)])}::text[])
|
|
22527
|
+
ORDER BY render_key COLLATE "C", theme COLLATE "C", size COLLATE "C"
|
|
21334
22528
|
`;
|
|
21335
|
-
return rows.
|
|
22529
|
+
return rows.map(parseArtifactBoardPreviewRow);
|
|
21336
22530
|
}
|
|
21337
|
-
async
|
|
21338
|
-
|
|
21339
|
-
|
|
21340
|
-
|
|
21341
|
-
|
|
21342
|
-
|
|
21343
|
-
|
|
21344
|
-
|
|
21345
|
-
|
|
21346
|
-
|
|
21347
|
-
|
|
21348
|
-
const affected = locked.map((row) => row.id);
|
|
21349
|
-
if (affected.length === 0)
|
|
21350
|
-
return 0;
|
|
21351
|
-
const deleted = await tx`
|
|
21352
|
-
DELETE FROM artifact_versions
|
|
21353
|
-
WHERE project_id = ${input.projectId} AND origin_run_id = ${input.runId}
|
|
21354
|
-
AND artifact_id IN ${tx(affected)}
|
|
21355
|
-
RETURNING origin_thread_id
|
|
21356
|
-
`;
|
|
21357
|
-
const threadId = deleted[0]?.origin_thread_id;
|
|
21358
|
-
await tx`
|
|
21359
|
-
DELETE FROM artifact_threads t
|
|
21360
|
-
WHERE t.project_id = ${input.projectId} AND t.artifact_id IN ${tx(affected)}
|
|
21361
|
-
AND t.thread_id = ${threadId ?? null} AND NOT t.copied
|
|
21362
|
-
AND NOT EXISTS (
|
|
21363
|
-
SELECT 1 FROM artifact_versions v
|
|
21364
|
-
WHERE v.project_id = t.project_id AND v.artifact_id = t.artifact_id
|
|
21365
|
-
AND v.origin_thread_id = t.thread_id
|
|
21366
|
-
)
|
|
21367
|
-
`;
|
|
21368
|
-
await tx`
|
|
21369
|
-
DELETE FROM artifacts a
|
|
21370
|
-
WHERE a.project_id = ${input.projectId} AND a.id IN ${tx(affected)}
|
|
21371
|
-
AND NOT EXISTS (
|
|
21372
|
-
SELECT 1 FROM artifact_versions v
|
|
21373
|
-
WHERE v.project_id = a.project_id AND v.artifact_id = a.id
|
|
22531
|
+
async putBoardPreviews(input) {
|
|
22532
|
+
if (input.previews.length === 0)
|
|
22533
|
+
return;
|
|
22534
|
+
await runPgTransaction(this.sql, async (tx) => {
|
|
22535
|
+
for (const preview of input.previews) {
|
|
22536
|
+
await tx`
|
|
22537
|
+
INSERT INTO artifact_board_previews (
|
|
22538
|
+
project_id, render_key, theme, size, file, created_at
|
|
22539
|
+
) VALUES (
|
|
22540
|
+
${input.projectId}, ${preview.renderKey}, ${preview.theme}, ${preview.size},
|
|
22541
|
+
${JSON.stringify(preview.file)}::text::jsonb, ${preview.createdAt}
|
|
21374
22542
|
)
|
|
22543
|
+
ON CONFLICT (project_id, render_key, theme, size) DO NOTHING
|
|
22544
|
+
`;
|
|
22545
|
+
}
|
|
22546
|
+
});
|
|
22547
|
+
}
|
|
22548
|
+
async createComment(input) {
|
|
22549
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
22550
|
+
if (!await lockArtifact(tx, input.projectId, input.artifactId)) {
|
|
22551
|
+
throw artifactStorageErrors.notFound(input.artifactId);
|
|
22552
|
+
}
|
|
22553
|
+
if (input.parentId !== undefined) {
|
|
22554
|
+
const [parent] = await tx`
|
|
22555
|
+
SELECT 1 FROM artifact_comments
|
|
22556
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22557
|
+
AND id = ${input.parentId} AND parent_id IS NULL
|
|
22558
|
+
`;
|
|
22559
|
+
if (!parent)
|
|
22560
|
+
throw artifactStorageErrors.commentNotFound(input.artifactId, input.parentId);
|
|
22561
|
+
}
|
|
22562
|
+
const [counted] = await tx`
|
|
22563
|
+
SELECT COUNT(*)::int AS total,
|
|
22564
|
+
(COUNT(*) FILTER (WHERE author_principal_type = 'guest'))::int AS guests
|
|
22565
|
+
FROM artifact_comments
|
|
22566
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
21375
22567
|
`;
|
|
21376
|
-
|
|
21377
|
-
|
|
21378
|
-
|
|
21379
|
-
|
|
21380
|
-
|
|
21381
|
-
|
|
21382
|
-
|
|
21383
|
-
|
|
21384
|
-
|
|
21385
|
-
|
|
22568
|
+
if ((counted?.total ?? 0) >= ARTIFACT_COMMENT_LIMITS.maxComments) {
|
|
22569
|
+
throw artifactStorageErrors.commentsFull(input.artifactId);
|
|
22570
|
+
}
|
|
22571
|
+
if (input.author.type === "guest" && (counted?.guests ?? 0) >= ARTIFACT_COMMENT_LIMITS.maxGuestComments) {
|
|
22572
|
+
throw artifactStorageErrors.guestCommentsFull(input.artifactId);
|
|
22573
|
+
}
|
|
22574
|
+
const [row] = await tx`
|
|
22575
|
+
INSERT INTO artifact_comments (
|
|
22576
|
+
project_id, artifact_id, id, parent_id, anchor, body, author_principal_type,
|
|
22577
|
+
author_principal_id, author_name, created_at, origin_thread_id, origin_run_id
|
|
22578
|
+
) VALUES (
|
|
22579
|
+
${input.projectId}, ${input.artifactId}, ${input.id}, ${input.parentId ?? null},
|
|
22580
|
+
${JSON.stringify(commentAnchor(input.anchor))}::text::jsonb, ${input.body},
|
|
22581
|
+
${input.author.type}, ${input.author.id},
|
|
22582
|
+
${input.author.type === "guest" ? input.author.name : null}, ${input.createdAt},
|
|
22583
|
+
${input.origin?.threadId ?? null}, ${input.origin?.runId ?? null}
|
|
22584
|
+
)
|
|
22585
|
+
RETURNING ${tx.unsafe(COMMENT_COLUMNS)}
|
|
21386
22586
|
`;
|
|
21387
|
-
return
|
|
22587
|
+
return parseArtifactCommentRow(row);
|
|
21388
22588
|
});
|
|
21389
22589
|
}
|
|
21390
|
-
|
|
21391
|
-
|
|
21392
|
-
|
|
21393
|
-
|
|
21394
|
-
|
|
21395
|
-
|
|
21396
|
-
)
|
|
21397
|
-
|
|
21398
|
-
|
|
21399
|
-
|
|
21400
|
-
${
|
|
21401
|
-
${input.
|
|
21402
|
-
|
|
21403
|
-
|
|
21404
|
-
|
|
21405
|
-
|
|
21406
|
-
|
|
21407
|
-
|
|
21408
|
-
|
|
22590
|
+
async getComment(input) {
|
|
22591
|
+
const [row] = await this.sql`
|
|
22592
|
+
SELECT ${this.sql.unsafe(COMMENT_COLUMNS)} FROM artifact_comments
|
|
22593
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22594
|
+
AND id = ${input.id}
|
|
22595
|
+
`;
|
|
22596
|
+
return row ? parseArtifactCommentRow(row) : null;
|
|
22597
|
+
}
|
|
22598
|
+
async listComments(input) {
|
|
22599
|
+
const rows = await this.sql`
|
|
22600
|
+
SELECT ${this.sql.unsafe(COMMENT_COLUMNS)} FROM artifact_comments
|
|
22601
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.id}
|
|
22602
|
+
ORDER BY created_at, seq
|
|
22603
|
+
`;
|
|
22604
|
+
return rows.map(parseArtifactCommentRow);
|
|
22605
|
+
}
|
|
22606
|
+
async resolveComment(input) {
|
|
22607
|
+
const [row] = await this.sql`
|
|
22608
|
+
UPDATE artifact_comments
|
|
22609
|
+
SET resolved_at = ${input.resolved?.at ?? null},
|
|
22610
|
+
resolved_by_principal_type = ${input.resolved?.by.type ?? null},
|
|
22611
|
+
resolved_by_principal_id = ${input.resolved?.by.id ?? null}
|
|
22612
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22613
|
+
AND id = ${input.id}
|
|
22614
|
+
RETURNING ${this.sql.unsafe(COMMENT_COLUMNS)}
|
|
22615
|
+
`;
|
|
22616
|
+
return row ? parseArtifactCommentRow(row) : null;
|
|
22617
|
+
}
|
|
22618
|
+
async deleteComment(input) {
|
|
22619
|
+
const rows = await this.sql`
|
|
22620
|
+
DELETE FROM artifact_comments
|
|
22621
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22622
|
+
AND id = ${input.id}
|
|
22623
|
+
RETURNING id
|
|
22624
|
+
`;
|
|
22625
|
+
return rows.length > 0;
|
|
22626
|
+
}
|
|
22627
|
+
async delete(input) {
|
|
22628
|
+
const rows = await this.sql`
|
|
22629
|
+
DELETE FROM artifacts WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
22630
|
+
RETURNING id
|
|
22631
|
+
`;
|
|
22632
|
+
return rows.length > 0;
|
|
22633
|
+
}
|
|
22634
|
+
}
|
|
22635
|
+
function commentAnchor(anchor) {
|
|
22636
|
+
return {
|
|
22637
|
+
...anchor.board === undefined ? {} : { board: anchor.board },
|
|
22638
|
+
...anchor.element === undefined ? {} : { element: anchor.element },
|
|
22639
|
+
...anchor.label === undefined ? {} : { label: anchor.label },
|
|
22640
|
+
...anchor.point === undefined ? {} : { point: { x: anchor.point.x, y: anchor.point.y } },
|
|
22641
|
+
revision: anchor.revision
|
|
22642
|
+
};
|
|
22643
|
+
}
|
|
22644
|
+
function documentJson(document) {
|
|
22645
|
+
return document ? JSON.stringify(document) : null;
|
|
22646
|
+
}
|
|
22647
|
+
async function lockArtifact(sql, projectId, artifactId) {
|
|
22648
|
+
const [row] = await sql`
|
|
22649
|
+
SELECT ${sql.unsafe(ARTIFACT_COLUMNS)}, next_version AS "nextVersion" FROM artifacts
|
|
22650
|
+
WHERE project_id = ${projectId} AND id = ${artifactId}
|
|
22651
|
+
FOR UPDATE
|
|
22652
|
+
`;
|
|
22653
|
+
return row ? { artifact: parseArtifactRow(row), nextVersion: Number(row.nextVersion) } : null;
|
|
22654
|
+
}
|
|
22655
|
+
async function latestVersion(sql, artifact) {
|
|
22656
|
+
const [row] = await sql`
|
|
22657
|
+
SELECT ${sql.unsafe(VERSION_COLUMNS)} FROM artifact_versions
|
|
22658
|
+
WHERE project_id = ${artifact.projectId} AND artifact_id = ${artifact.id}
|
|
22659
|
+
AND version = ${artifact.latestVersion}
|
|
22660
|
+
`;
|
|
22661
|
+
if (!row)
|
|
22662
|
+
throw new Error(`[SixbPg] Artifact '${artifact.id}' has no latest version.`);
|
|
22663
|
+
return parseArtifactVersionRow(row);
|
|
22664
|
+
}
|
|
22665
|
+
function assertBase(artifact, latest, baseRevision) {
|
|
22666
|
+
if (baseRevision === undefined || baseRevision === artifact.revision)
|
|
22667
|
+
return;
|
|
22668
|
+
throw artifactStorageErrors.moved(artifact.id, baseRevision, {
|
|
22669
|
+
version: latest.version,
|
|
22670
|
+
revision: artifact.revision,
|
|
22671
|
+
author: latest.author,
|
|
22672
|
+
...latest.frozenBy ? { frozenBy: latest.frozenBy } : {}
|
|
22673
|
+
});
|
|
22674
|
+
}
|
|
22675
|
+
async function writeLatest(sql, { artifact, nextVersion }, latest, content, document) {
|
|
22676
|
+
const { projectId, id: artifactId } = artifact;
|
|
22677
|
+
const revision = artifact.revision + 1;
|
|
22678
|
+
const state = documentJson(document);
|
|
22679
|
+
if (latest.frozenAt) {
|
|
22680
|
+
const version = await insertVersion(sql, {
|
|
22681
|
+
projectId,
|
|
22682
|
+
artifactId,
|
|
22683
|
+
version: nextVersion,
|
|
22684
|
+
revision,
|
|
22685
|
+
content
|
|
22686
|
+
});
|
|
22687
|
+
const [row] = await sql`
|
|
22688
|
+
UPDATE artifacts
|
|
22689
|
+
SET title = ${content.title}, latest_version = ${version.version},
|
|
22690
|
+
next_version = ${version.version + 1}, revision = ${revision},
|
|
22691
|
+
updated_at = ${content.updatedAt},
|
|
22692
|
+
document = COALESCE(${state}::text::jsonb, document),
|
|
22693
|
+
document_revision = CASE WHEN ${state}::text IS NULL THEN document_revision
|
|
22694
|
+
ELSE ${revision}::int END
|
|
22695
|
+
WHERE project_id = ${projectId} AND id = ${artifactId}
|
|
22696
|
+
RETURNING ${sql.unsafe(ARTIFACT_COLUMNS)}
|
|
22697
|
+
`;
|
|
22698
|
+
return { artifact: parseArtifactRow(row), version };
|
|
22699
|
+
}
|
|
22700
|
+
const version = await replaceContent(sql, {
|
|
22701
|
+
projectId,
|
|
22702
|
+
artifactId,
|
|
22703
|
+
version: latest.version,
|
|
22704
|
+
revision,
|
|
22705
|
+
content
|
|
22706
|
+
});
|
|
22707
|
+
const [row] = await sql`
|
|
22708
|
+
UPDATE artifacts
|
|
22709
|
+
SET title = ${content.title}, revision = ${revision}, updated_at = ${content.updatedAt},
|
|
22710
|
+
document = COALESCE(${state}::text::jsonb, document),
|
|
22711
|
+
document_revision = CASE WHEN ${state}::text IS NULL THEN document_revision
|
|
22712
|
+
ELSE ${revision}::int END
|
|
22713
|
+
WHERE project_id = ${projectId} AND id = ${artifactId}
|
|
22714
|
+
RETURNING ${sql.unsafe(ARTIFACT_COLUMNS)}
|
|
22715
|
+
`;
|
|
22716
|
+
return { artifact: parseArtifactRow(row), version };
|
|
22717
|
+
}
|
|
22718
|
+
async function replaceContent(sql, input) {
|
|
22719
|
+
const { content } = input;
|
|
22720
|
+
const [row] = await sql`
|
|
22721
|
+
UPDATE artifact_versions
|
|
22722
|
+
SET revision = ${input.revision}, title = ${content.title}, entry = ${content.entry},
|
|
22723
|
+
files = ${JSON.stringify(content.files)}::text::jsonb,
|
|
22724
|
+
outline = ${JSON.stringify(content.outline)}::text::jsonb,
|
|
22725
|
+
refs = ${JSON.stringify(content.references)}::text::jsonb,
|
|
22726
|
+
author_principal_type = ${content.author.type},
|
|
22727
|
+
author_principal_id = ${content.author.id},
|
|
22728
|
+
origin_thread_id = ${content.origin?.threadId ?? null},
|
|
22729
|
+
origin_run_id = ${content.origin?.runId ?? null}, updated_at = ${content.updatedAt}
|
|
22730
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22731
|
+
AND version = ${input.version}
|
|
22732
|
+
RETURNING ${sql.unsafe(VERSION_COLUMNS)}
|
|
22733
|
+
`;
|
|
22734
|
+
return parseArtifactVersionRow(row);
|
|
22735
|
+
}
|
|
22736
|
+
async function insertVersion(sql, input) {
|
|
22737
|
+
const { content } = input;
|
|
22738
|
+
const [row] = await sql`
|
|
22739
|
+
INSERT INTO artifact_versions (
|
|
22740
|
+
project_id, artifact_id, version, revision, title, entry, files, outline, refs,
|
|
22741
|
+
author_principal_type, author_principal_id, origin_thread_id, origin_run_id, created_at,
|
|
22742
|
+
updated_at
|
|
22743
|
+
) VALUES (
|
|
22744
|
+
${input.projectId}, ${input.artifactId}, ${input.version}, ${input.revision},
|
|
22745
|
+
${content.title}, ${content.entry},
|
|
22746
|
+
${JSON.stringify(content.files)}::text::jsonb,
|
|
22747
|
+
${JSON.stringify(content.outline)}::text::jsonb,
|
|
22748
|
+
${JSON.stringify(content.references)}::text::jsonb,
|
|
22749
|
+
${content.author.type}, ${content.author.id},
|
|
22750
|
+
${content.origin?.threadId ?? null}, ${content.origin?.runId ?? null},
|
|
22751
|
+
${content.updatedAt}, ${content.updatedAt}
|
|
22752
|
+
)
|
|
22753
|
+
RETURNING ${sql.unsafe(VERSION_COLUMNS)}
|
|
22754
|
+
`;
|
|
21409
22755
|
return parseArtifactVersionRow(row);
|
|
21410
22756
|
}
|
|
21411
|
-
async function
|
|
22757
|
+
async function keepRestorePoint(sql, latest, input) {
|
|
22758
|
+
const { projectId, artifactId } = latest;
|
|
22759
|
+
await sql`
|
|
22760
|
+
INSERT INTO artifact_restore_points (
|
|
22761
|
+
project_id, artifact_id, revision, version, frozen, title, entry, files, outline, refs,
|
|
22762
|
+
author_principal_type, author_principal_id, origin_thread_id, origin_run_id, updated_at,
|
|
22763
|
+
reason, turn_run_id, created_at
|
|
22764
|
+
)
|
|
22765
|
+
SELECT project_id, artifact_id, revision, version, frozen_at IS NOT NULL, title, entry, files,
|
|
22766
|
+
outline, refs, author_principal_type, author_principal_id, origin_thread_id,
|
|
22767
|
+
origin_run_id, updated_at, ${input.reason}, ${input.turnRunId ?? null},
|
|
22768
|
+
${input.createdAt}
|
|
22769
|
+
FROM artifact_versions
|
|
22770
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId} AND version = ${latest.version}
|
|
22771
|
+
`;
|
|
22772
|
+
await sql`
|
|
22773
|
+
DELETE FROM artifact_restore_points
|
|
22774
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId} AND revision NOT IN (
|
|
22775
|
+
SELECT revision FROM artifact_restore_points
|
|
22776
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId}
|
|
22777
|
+
ORDER BY revision DESC LIMIT ${ARTIFACT_RESTORE_POINTS}
|
|
22778
|
+
)
|
|
22779
|
+
`;
|
|
22780
|
+
}
|
|
22781
|
+
async function turnWrote(sql, artifact, turnRunId) {
|
|
22782
|
+
const rows = await sql`
|
|
22783
|
+
SELECT 1 FROM artifact_turn_writes
|
|
22784
|
+
WHERE project_id = ${artifact.projectId} AND artifact_id = ${artifact.id}
|
|
22785
|
+
AND turn_run_id = ${turnRunId}
|
|
22786
|
+
`;
|
|
22787
|
+
return rows.length > 0;
|
|
22788
|
+
}
|
|
22789
|
+
async function recordTurnWrite(sql, input) {
|
|
22790
|
+
const { projectId, artifactId } = input;
|
|
22791
|
+
await sql`
|
|
22792
|
+
INSERT INTO artifact_turn_writes
|
|
22793
|
+
(project_id, artifact_id, turn_run_id, created, revision, writes)
|
|
22794
|
+
VALUES (${projectId}, ${artifactId}, ${input.turnRunId}, ${input.created}, ${input.revision}, 1)
|
|
22795
|
+
ON CONFLICT (project_id, artifact_id, turn_run_id) DO UPDATE SET
|
|
22796
|
+
revision = excluded.revision, writes = artifact_turn_writes.writes + 1
|
|
22797
|
+
`;
|
|
21412
22798
|
await sql`
|
|
21413
|
-
|
|
21414
|
-
|
|
22799
|
+
DELETE FROM artifact_turn_writes
|
|
22800
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId} AND turn_run_id NOT IN (
|
|
22801
|
+
SELECT turn_run_id FROM artifact_turn_writes
|
|
22802
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId}
|
|
22803
|
+
ORDER BY revision DESC LIMIT ${ARTIFACT_RESTORE_POINTS}
|
|
22804
|
+
)
|
|
22805
|
+
`;
|
|
22806
|
+
}
|
|
22807
|
+
async function undoTurn(sql, projectId, write, turnRunId, document) {
|
|
22808
|
+
const { artifactId } = write;
|
|
22809
|
+
const [row] = await sql`
|
|
22810
|
+
SELECT ${sql.unsafe(RESTORE_POINT_COLUMNS)} FROM artifact_restore_points
|
|
22811
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId}
|
|
22812
|
+
AND turn_run_id = ${turnRunId}
|
|
22813
|
+
`;
|
|
22814
|
+
if (!row)
|
|
22815
|
+
return null;
|
|
22816
|
+
const point = parseArtifactRestorePointRow(row);
|
|
22817
|
+
const [current] = await sql`
|
|
22818
|
+
SELECT revision FROM artifacts WHERE project_id = ${projectId} AND id = ${artifactId}
|
|
22819
|
+
`;
|
|
22820
|
+
const [target] = await sql`
|
|
22821
|
+
SELECT frozen_at AS "frozenAt" FROM artifact_versions
|
|
22822
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId} AND version = ${point.version}
|
|
22823
|
+
`;
|
|
22824
|
+
if (!current || !target)
|
|
22825
|
+
return null;
|
|
22826
|
+
const untouched = turnUndoable({
|
|
22827
|
+
write,
|
|
22828
|
+
revision: Number(current.revision),
|
|
22829
|
+
point,
|
|
22830
|
+
target: { frozen: target.frozenAt !== null }
|
|
22831
|
+
});
|
|
22832
|
+
if (!untouched)
|
|
22833
|
+
return null;
|
|
22834
|
+
const revision = Number(current.revision) + 1;
|
|
22835
|
+
await sql`
|
|
22836
|
+
DELETE FROM artifact_versions
|
|
22837
|
+
WHERE project_id = ${projectId} AND artifact_id = ${artifactId} AND version > ${point.version}
|
|
22838
|
+
`;
|
|
22839
|
+
const at = { projectId, artifactId, version: point.version, revision };
|
|
22840
|
+
const undone = point.frozen ? await freshRevision(sql, at) : await replaceContent(sql, {
|
|
22841
|
+
...at,
|
|
22842
|
+
content: {
|
|
22843
|
+
title: point.title,
|
|
22844
|
+
entry: point.entry,
|
|
22845
|
+
files: point.files,
|
|
22846
|
+
outline: point.outline,
|
|
22847
|
+
references: point.references,
|
|
22848
|
+
author: point.author,
|
|
22849
|
+
...point.origin ? { origin: point.origin } : {},
|
|
22850
|
+
updatedAt: point.updatedAt
|
|
22851
|
+
}
|
|
22852
|
+
});
|
|
22853
|
+
const updatedAt = new Date(Math.max(undone.updatedAt.getTime(), undone.frozenAt?.getTime() ?? 0));
|
|
22854
|
+
const [artifact] = await sql`
|
|
22855
|
+
UPDATE artifacts
|
|
22856
|
+
SET title = ${undone.title}, latest_version = ${undone.version}, revision = ${revision},
|
|
22857
|
+
updated_at = ${updatedAt},
|
|
22858
|
+
document = COALESCE(${documentJson(document)}::text::jsonb, document),
|
|
22859
|
+
document_revision = CASE WHEN ${documentJson(document)}::text IS NULL THEN document_revision
|
|
22860
|
+
ELSE ${revision}::int END
|
|
22861
|
+
WHERE project_id = ${projectId} AND id = ${artifactId}
|
|
22862
|
+
RETURNING ${sql.unsafe(ARTIFACT_COLUMNS)}
|
|
22863
|
+
`;
|
|
22864
|
+
return { artifact: parseArtifactRow(artifact), version: undone };
|
|
22865
|
+
}
|
|
22866
|
+
async function freshRevision(sql, input) {
|
|
22867
|
+
const [row] = await sql`
|
|
22868
|
+
UPDATE artifact_versions SET revision = ${input.revision}
|
|
22869
|
+
WHERE project_id = ${input.projectId} AND artifact_id = ${input.artifactId}
|
|
22870
|
+
AND version = ${input.version}
|
|
22871
|
+
RETURNING ${sql.unsafe(VERSION_COLUMNS)}
|
|
22872
|
+
`;
|
|
22873
|
+
return parseArtifactVersionRow(row);
|
|
22874
|
+
}
|
|
22875
|
+
async function shareIn(sql, projectId, artifactId, content, turnRunId) {
|
|
22876
|
+
if (!content.origin)
|
|
22877
|
+
return;
|
|
22878
|
+
await sql`
|
|
22879
|
+
INSERT INTO artifact_threads (project_id, artifact_id, thread_id, shared_at, linked_by_run_id)
|
|
22880
|
+
VALUES (
|
|
22881
|
+
${projectId}, ${artifactId}, ${content.origin.threadId}, ${content.updatedAt},
|
|
22882
|
+
${turnRunId ?? null}
|
|
22883
|
+
)
|
|
21415
22884
|
ON CONFLICT (project_id, artifact_id, thread_id) DO NOTHING
|
|
21416
22885
|
`;
|
|
21417
22886
|
}
|
|
@@ -21425,7 +22894,8 @@ var DEFAULT_CONNECT_TIMEOUT_SECONDS = 10;
|
|
|
21425
22894
|
function createPgClient(options) {
|
|
21426
22895
|
const connection = {
|
|
21427
22896
|
search_path: options.schemaName,
|
|
21428
|
-
enable_incremental_sort: "off"
|
|
22897
|
+
enable_incremental_sort: "off",
|
|
22898
|
+
jit: "off"
|
|
21429
22899
|
};
|
|
21430
22900
|
if (options.statementTimeoutMillis !== undefined) {
|
|
21431
22901
|
connection.statement_timeout = options.statementTimeoutMillis;
|
|
@@ -21529,6 +22999,221 @@ function assertDirectTlsTarget(sql, options) {
|
|
|
21529
22999
|
return sql;
|
|
21530
23000
|
}
|
|
21531
23001
|
|
|
23002
|
+
// src/pg-dataset-ingest-storage.ts
|
|
23003
|
+
import {
|
|
23004
|
+
DurableDatasetIngestStorage,
|
|
23005
|
+
initialDatasetIngestState
|
|
23006
|
+
} from "@sixb/core/internal/dataset-ingest-storage-provider";
|
|
23007
|
+
class PgDatasetIngestStorage extends DurableDatasetIngestStorage {
|
|
23008
|
+
constructor(sql) {
|
|
23009
|
+
super(new PgDatasetIngestBackend(sql));
|
|
23010
|
+
}
|
|
23011
|
+
}
|
|
23012
|
+
|
|
23013
|
+
class PgDatasetIngestBackend {
|
|
23014
|
+
sql;
|
|
23015
|
+
constructor(sql) {
|
|
23016
|
+
this.sql = sql;
|
|
23017
|
+
}
|
|
23018
|
+
read(run) {
|
|
23019
|
+
return run(new PgDatasetIngestPersistence(this.sql));
|
|
23020
|
+
}
|
|
23021
|
+
transaction(run) {
|
|
23022
|
+
return runPgTransaction(this.sql, (tx) => run(new PgDatasetIngestPersistence(tx)));
|
|
23023
|
+
}
|
|
23024
|
+
}
|
|
23025
|
+
|
|
23026
|
+
class PgDatasetIngestPersistence {
|
|
23027
|
+
sql;
|
|
23028
|
+
constructor(sql) {
|
|
23029
|
+
this.sql = sql;
|
|
23030
|
+
}
|
|
23031
|
+
async now() {
|
|
23032
|
+
const [row] = await this.sql`SELECT clock_timestamp() AS now`;
|
|
23033
|
+
if (!row)
|
|
23034
|
+
throw new Error("[SixbPg] Database clock returned no row.");
|
|
23035
|
+
return new Date(row.now);
|
|
23036
|
+
}
|
|
23037
|
+
async lockState(projectId, datasetId) {
|
|
23038
|
+
await this.sql`
|
|
23039
|
+
INSERT INTO dataset_ingest_state (project_id, dataset_id)
|
|
23040
|
+
VALUES (${projectId}, ${datasetId})
|
|
23041
|
+
ON CONFLICT (project_id, dataset_id) DO NOTHING
|
|
23042
|
+
`;
|
|
23043
|
+
const [row] = await this.sql`
|
|
23044
|
+
SELECT dataset_id, request::text AS request, requested_at, lease_owner, lease_expires_at,
|
|
23045
|
+
flush_id, flush_started_at, failure_streak, last_flushed_at
|
|
23046
|
+
FROM dataset_ingest_state
|
|
23047
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId}
|
|
23048
|
+
FOR UPDATE
|
|
23049
|
+
`;
|
|
23050
|
+
return row ? stateFromRow(row) : initialDatasetIngestState(datasetId);
|
|
23051
|
+
}
|
|
23052
|
+
async writeState(projectId, state) {
|
|
23053
|
+
await this.sql`
|
|
23054
|
+
UPDATE dataset_ingest_state
|
|
23055
|
+
SET request = ${state.request},
|
|
23056
|
+
requested_at = ${state.requestedAt},
|
|
23057
|
+
lease_owner = ${state.leaseOwner},
|
|
23058
|
+
lease_expires_at = ${state.leaseExpiresAt},
|
|
23059
|
+
flush_id = ${state.flushId},
|
|
23060
|
+
flush_started_at = ${state.flushStartedAt},
|
|
23061
|
+
failure_streak = ${state.failureStreak},
|
|
23062
|
+
last_flushed_at = ${state.lastFlushedAt}
|
|
23063
|
+
WHERE project_id = ${projectId} AND dataset_id = ${state.datasetId}
|
|
23064
|
+
`;
|
|
23065
|
+
}
|
|
23066
|
+
async insertDelivery(projectId, record) {
|
|
23067
|
+
const [row] = await this.sql`
|
|
23068
|
+
INSERT INTO dataset_ingest_deliveries (
|
|
23069
|
+
project_id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
23070
|
+
definition_hash, changes
|
|
23071
|
+
) VALUES (
|
|
23072
|
+
${projectId}, ${record.datasetId}, ${record.executionId}, ${record.correlationId},
|
|
23073
|
+
${record.receivedAt}, ${record.changeCount}, ${record.definitionHash}, ${record.changes}
|
|
23074
|
+
)
|
|
23075
|
+
RETURNING id::text AS id
|
|
23076
|
+
`;
|
|
23077
|
+
if (!row)
|
|
23078
|
+
throw new Error("[SixbPg] Dataset ingest delivery insert returned no id.");
|
|
23079
|
+
return row.id;
|
|
23080
|
+
}
|
|
23081
|
+
async listUntaken(projectId, datasetId, limit) {
|
|
23082
|
+
const rows = await this.sql`
|
|
23083
|
+
-- The id is read as text: ORDER BY names the table's column, not that alias.
|
|
23084
|
+
SELECT id::text AS id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
23085
|
+
definition_hash
|
|
23086
|
+
FROM dataset_ingest_deliveries
|
|
23087
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id IS NULL
|
|
23088
|
+
AND rejected_at IS NULL
|
|
23089
|
+
ORDER BY dataset_ingest_deliveries.id
|
|
23090
|
+
LIMIT ${limit}
|
|
23091
|
+
`;
|
|
23092
|
+
return rows.map(deliveryFromRow);
|
|
23093
|
+
}
|
|
23094
|
+
async take(projectId, datasetId, deliveryIds, flushId) {
|
|
23095
|
+
await this.sql`
|
|
23096
|
+
UPDATE dataset_ingest_deliveries
|
|
23097
|
+
SET flush_id = ${flushId}
|
|
23098
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId}
|
|
23099
|
+
AND id = ANY(${this.sql.array([...deliveryIds])}::bigint[])
|
|
23100
|
+
`;
|
|
23101
|
+
}
|
|
23102
|
+
async listFlush(projectId, datasetId, flushId) {
|
|
23103
|
+
const rows = await this.sql`
|
|
23104
|
+
SELECT id::text AS id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
23105
|
+
definition_hash
|
|
23106
|
+
FROM dataset_ingest_deliveries
|
|
23107
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id = ${flushId}
|
|
23108
|
+
ORDER BY dataset_ingest_deliveries.id
|
|
23109
|
+
`;
|
|
23110
|
+
return rows.map(deliveryFromRow);
|
|
23111
|
+
}
|
|
23112
|
+
async readFlush(projectId, datasetId, flushId, page) {
|
|
23113
|
+
const rows = await this.sql`
|
|
23114
|
+
SELECT id::text AS id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
23115
|
+
definition_hash, changes
|
|
23116
|
+
FROM dataset_ingest_deliveries
|
|
23117
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id = ${flushId}
|
|
23118
|
+
AND id > ${page.afterId ?? "0"}::bigint
|
|
23119
|
+
ORDER BY dataset_ingest_deliveries.id
|
|
23120
|
+
LIMIT ${page.limit}
|
|
23121
|
+
`;
|
|
23122
|
+
return rows.map((row) => ({ delivery: deliveryFromRow(row), changes: row.changes }));
|
|
23123
|
+
}
|
|
23124
|
+
async deleteFlush(projectId, datasetId, flushId) {
|
|
23125
|
+
await this.sql`
|
|
23126
|
+
DELETE FROM dataset_ingest_deliveries
|
|
23127
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id = ${flushId}
|
|
23128
|
+
`;
|
|
23129
|
+
}
|
|
23130
|
+
async rejectDelivery(projectId, datasetId, flushId, deliveryId, rejection) {
|
|
23131
|
+
await this.sql`
|
|
23132
|
+
UPDATE dataset_ingest_deliveries
|
|
23133
|
+
SET flush_id = NULL, rejected_at = ${rejection.rejectedAt}, rejection = ${rejection.reason}
|
|
23134
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id = ${flushId}
|
|
23135
|
+
AND id = ${deliveryId}::bigint
|
|
23136
|
+
`;
|
|
23137
|
+
}
|
|
23138
|
+
async listRejected(projectId, datasetId, limit) {
|
|
23139
|
+
const rows = await this.sql`
|
|
23140
|
+
SELECT id::text AS id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
23141
|
+
definition_hash, changes, rejected_at, rejection
|
|
23142
|
+
FROM dataset_ingest_deliveries
|
|
23143
|
+
WHERE project_id = ${projectId} AND rejected_at IS NOT NULL
|
|
23144
|
+
AND (${datasetId === undefined} OR dataset_id = ${datasetId ?? ""})
|
|
23145
|
+
ORDER BY rejected_at DESC, dataset_ingest_deliveries.id DESC
|
|
23146
|
+
LIMIT ${limit}
|
|
23147
|
+
`;
|
|
23148
|
+
return rows.map((row) => ({
|
|
23149
|
+
delivery: deliveryFromRow(row),
|
|
23150
|
+
rejectedAt: new Date(row.rejected_at),
|
|
23151
|
+
reason: row.rejection,
|
|
23152
|
+
changes: row.changes
|
|
23153
|
+
}));
|
|
23154
|
+
}
|
|
23155
|
+
async deleteRejected(projectId, before) {
|
|
23156
|
+
await this.sql`
|
|
23157
|
+
DELETE FROM dataset_ingest_deliveries
|
|
23158
|
+
WHERE project_id = ${projectId} AND rejected_at < ${before}
|
|
23159
|
+
`;
|
|
23160
|
+
}
|
|
23161
|
+
async listStates(projectId, datasetIds) {
|
|
23162
|
+
const rows = await this.sql`
|
|
23163
|
+
SELECT dataset_id, request::text AS request, requested_at, lease_owner, lease_expires_at,
|
|
23164
|
+
flush_id, flush_started_at, failure_streak, last_flushed_at
|
|
23165
|
+
FROM dataset_ingest_state
|
|
23166
|
+
WHERE project_id = ${projectId}
|
|
23167
|
+
AND (${datasetIds === undefined} OR dataset_id = ANY(${this.sql.array([...datasetIds ?? []])}::text[]))
|
|
23168
|
+
`;
|
|
23169
|
+
return rows.map(stateFromRow);
|
|
23170
|
+
}
|
|
23171
|
+
async countPending(projectId, datasetIds) {
|
|
23172
|
+
const rows = await this.sql`
|
|
23173
|
+
SELECT dataset_id,
|
|
23174
|
+
count(*) FILTER (WHERE rejected_at IS NULL)::text AS deliveries,
|
|
23175
|
+
coalesce(sum(change_count) FILTER (WHERE rejected_at IS NULL), 0)::text AS changes,
|
|
23176
|
+
min(received_at) FILTER (WHERE rejected_at IS NULL) AS oldest,
|
|
23177
|
+
count(*) FILTER (WHERE rejected_at IS NOT NULL)::text AS rejected
|
|
23178
|
+
FROM dataset_ingest_deliveries
|
|
23179
|
+
WHERE project_id = ${projectId}
|
|
23180
|
+
AND (${datasetIds === undefined} OR dataset_id = ANY(${this.sql.array([...datasetIds ?? []])}::text[]))
|
|
23181
|
+
GROUP BY dataset_id
|
|
23182
|
+
`;
|
|
23183
|
+
return rows.map((row) => ({
|
|
23184
|
+
datasetId: row.dataset_id,
|
|
23185
|
+
pendingDeliveries: Number(row.deliveries),
|
|
23186
|
+
pendingChanges: Number(row.changes),
|
|
23187
|
+
oldestReceivedAt: row.oldest === null ? null : new Date(row.oldest),
|
|
23188
|
+
rejectedDeliveries: Number(row.rejected)
|
|
23189
|
+
}));
|
|
23190
|
+
}
|
|
23191
|
+
}
|
|
23192
|
+
function stateFromRow(row) {
|
|
23193
|
+
return {
|
|
23194
|
+
datasetId: row.dataset_id,
|
|
23195
|
+
request: Number(row.request),
|
|
23196
|
+
requestedAt: row.requested_at === null ? null : new Date(row.requested_at),
|
|
23197
|
+
leaseOwner: row.lease_owner,
|
|
23198
|
+
leaseExpiresAt: row.lease_expires_at === null ? null : new Date(row.lease_expires_at),
|
|
23199
|
+
flushId: row.flush_id,
|
|
23200
|
+
flushStartedAt: row.flush_started_at === null ? null : new Date(row.flush_started_at),
|
|
23201
|
+
failureStreak: row.failure_streak,
|
|
23202
|
+
lastFlushedAt: row.last_flushed_at === null ? null : new Date(row.last_flushed_at)
|
|
23203
|
+
};
|
|
23204
|
+
}
|
|
23205
|
+
function deliveryFromRow(row) {
|
|
23206
|
+
return {
|
|
23207
|
+
id: row.id,
|
|
23208
|
+
datasetId: row.dataset_id,
|
|
23209
|
+
executionId: row.execution_id,
|
|
23210
|
+
correlationId: row.correlation_id,
|
|
23211
|
+
receivedAt: new Date(row.received_at),
|
|
23212
|
+
changeCount: row.change_count,
|
|
23213
|
+
definitionHash: row.definition_hash
|
|
23214
|
+
};
|
|
23215
|
+
}
|
|
23216
|
+
|
|
21532
23217
|
// src/pg-file-download-grant-storage.ts
|
|
21533
23218
|
import { parseFileDownloadGrantRow } from "@sixb/core/internal/storage";
|
|
21534
23219
|
|
|
@@ -21717,12 +23402,88 @@ function jsonText(value) {
|
|
|
21717
23402
|
return value === undefined ? null : JSON.stringify(value);
|
|
21718
23403
|
}
|
|
21719
23404
|
|
|
23405
|
+
// src/pg-lake-retention-run-storage.ts
|
|
23406
|
+
class PgLakeRetentionRunStorage {
|
|
23407
|
+
sql;
|
|
23408
|
+
constructor(sql) {
|
|
23409
|
+
this.sql = sql;
|
|
23410
|
+
}
|
|
23411
|
+
async record(run) {
|
|
23412
|
+
await this.sql`
|
|
23413
|
+
INSERT INTO lake_retention_runs (project_id, started_at, finished_at, failed)
|
|
23414
|
+
VALUES (${run.projectId}, ${run.startedAt}, ${run.finishedAt}, ${run.failed})
|
|
23415
|
+
ON CONFLICT (project_id) DO UPDATE SET
|
|
23416
|
+
started_at = EXCLUDED.started_at,
|
|
23417
|
+
finished_at = EXCLUDED.finished_at,
|
|
23418
|
+
failed = EXCLUDED.failed
|
|
23419
|
+
`;
|
|
23420
|
+
}
|
|
23421
|
+
async getLast(input) {
|
|
23422
|
+
const [row] = await this.sql`
|
|
23423
|
+
SELECT project_id, started_at, finished_at, failed
|
|
23424
|
+
FROM lake_retention_runs
|
|
23425
|
+
WHERE project_id = ${input.projectId}
|
|
23426
|
+
`;
|
|
23427
|
+
if (!row)
|
|
23428
|
+
return null;
|
|
23429
|
+
return {
|
|
23430
|
+
projectId: row.project_id,
|
|
23431
|
+
startedAt: new Date(row.started_at),
|
|
23432
|
+
finishedAt: row.finished_at ? new Date(row.finished_at) : null,
|
|
23433
|
+
failed: row.failed
|
|
23434
|
+
};
|
|
23435
|
+
}
|
|
23436
|
+
}
|
|
23437
|
+
|
|
23438
|
+
// src/pg-maintenance-heartbeat-storage.ts
|
|
23439
|
+
import {
|
|
23440
|
+
parseMaintenanceHeartbeatRow,
|
|
23441
|
+
serializeMaintenancePasses
|
|
23442
|
+
} from "@sixb/core/internal/storage";
|
|
23443
|
+
|
|
23444
|
+
class PgMaintenanceHeartbeatStorage {
|
|
23445
|
+
sql;
|
|
23446
|
+
constructor(sql) {
|
|
23447
|
+
this.sql = sql;
|
|
23448
|
+
}
|
|
23449
|
+
async record(heartbeat) {
|
|
23450
|
+
const passes = serializeMaintenancePasses(heartbeat.passes);
|
|
23451
|
+
await this.sql`
|
|
23452
|
+
INSERT INTO maintenance_heartbeats (project_id, owner, beat_at, every_ms, passes)
|
|
23453
|
+
VALUES (
|
|
23454
|
+
${heartbeat.projectId}, ${heartbeat.owner}, ${heartbeat.beatAt}, ${heartbeat.everyMs},
|
|
23455
|
+
${passes}::text::jsonb
|
|
23456
|
+
)
|
|
23457
|
+
ON CONFLICT (project_id, owner) DO UPDATE SET
|
|
23458
|
+
beat_at = EXCLUDED.beat_at,
|
|
23459
|
+
every_ms = EXCLUDED.every_ms,
|
|
23460
|
+
passes = EXCLUDED.passes
|
|
23461
|
+
`;
|
|
23462
|
+
}
|
|
23463
|
+
async list(input) {
|
|
23464
|
+
const rows = await this.sql`
|
|
23465
|
+
SELECT project_id, owner, beat_at, every_ms, passes
|
|
23466
|
+
FROM maintenance_heartbeats
|
|
23467
|
+
WHERE project_id = ${input.projectId}
|
|
23468
|
+
`;
|
|
23469
|
+
return rows.map((row) => parseMaintenanceHeartbeatRow({
|
|
23470
|
+
projectId: row.project_id,
|
|
23471
|
+
owner: row.owner,
|
|
23472
|
+
beatAt: row.beat_at,
|
|
23473
|
+
everyMs: row.every_ms,
|
|
23474
|
+
passes: row.passes
|
|
23475
|
+
}));
|
|
23476
|
+
}
|
|
23477
|
+
}
|
|
23478
|
+
|
|
21720
23479
|
// src/pg-pipeline-run-storage.ts
|
|
21721
23480
|
import { parseSixbFailure as parseSixbFailure4, serializeSixbFailure as serializeSixbFailure3 } from "@sixb/core/internal/errors";
|
|
21722
23481
|
import { assertPipelineRunExecution } from "@sixb/core/internal/pipeline-run-storage-provider";
|
|
21723
23482
|
import {
|
|
21724
23483
|
canFinishPipelineRun,
|
|
21725
|
-
canRequeuePipelineRunAfterEnqueueFailure
|
|
23484
|
+
canRequeuePipelineRunAfterEnqueueFailure,
|
|
23485
|
+
holdsRun,
|
|
23486
|
+
mayFinishRun
|
|
21726
23487
|
} from "@sixb/core/internal/storage";
|
|
21727
23488
|
import { PIPELINE_RUN_FAILURE_CODES, PipelineRunError } from "@sixb/core/storage";
|
|
21728
23489
|
|
|
@@ -21753,7 +23514,99 @@ async function queryLatestRunsByOwnerId(input) {
|
|
|
21753
23514
|
});
|
|
21754
23515
|
}
|
|
21755
23516
|
|
|
23517
|
+
// src/run-claim-sql.ts
|
|
23518
|
+
import { laterLease, planRunClaim } from "@sixb/core/internal/storage";
|
|
23519
|
+
function pgRunExecutionFields(row) {
|
|
23520
|
+
return {
|
|
23521
|
+
attempt: Number(row.attempt),
|
|
23522
|
+
execution: row.execution_token && row.execution_queue_lease_expires_at ? {
|
|
23523
|
+
token: row.execution_token,
|
|
23524
|
+
queueLeaseExpiresAt: new Date(row.execution_queue_lease_expires_at)
|
|
23525
|
+
} : undefined
|
|
23526
|
+
};
|
|
23527
|
+
}
|
|
23528
|
+
async function lockPgRun(tx, table, projectId, id) {
|
|
23529
|
+
const [row] = await tx.unsafe(`SELECT ${table.returning} FROM ${table.name} WHERE project_id = $1 AND id = $2 FOR UPDATE`, [projectId, id]);
|
|
23530
|
+
if (!row)
|
|
23531
|
+
throw table.notFound(projectId, id);
|
|
23532
|
+
return table.toRun(row);
|
|
23533
|
+
}
|
|
23534
|
+
async function claimPgRun(tx, table, input, sibling) {
|
|
23535
|
+
const existing = await lockPgRun(tx, table, input.projectId, input.id);
|
|
23536
|
+
const plan = planRunClaim(existing, input, sibling);
|
|
23537
|
+
if (!plan.take)
|
|
23538
|
+
return plan.claim;
|
|
23539
|
+
const [updated] = await tx.unsafe(`UPDATE ${table.name}
|
|
23540
|
+
SET status = 'running', started_at = $1, attempt = $2, execution_token = $3,
|
|
23541
|
+
execution_queue_lease_expires_at = $4, error = NULL
|
|
23542
|
+
WHERE project_id = $5 AND id = $6
|
|
23543
|
+
RETURNING ${table.returning}`, [
|
|
23544
|
+
plan.startedAt,
|
|
23545
|
+
plan.attempt,
|
|
23546
|
+
plan.execution.token,
|
|
23547
|
+
plan.execution.queueLeaseExpiresAt,
|
|
23548
|
+
input.projectId,
|
|
23549
|
+
input.id
|
|
23550
|
+
]);
|
|
23551
|
+
if (!updated)
|
|
23552
|
+
throw table.notFound(input.projectId, input.id);
|
|
23553
|
+
return { outcome: plan.outcome, run: table.toRun(updated) };
|
|
23554
|
+
}
|
|
23555
|
+
async function lockHeldPgRun(tx, table, input) {
|
|
23556
|
+
const run = await lockPgRun(tx, table, input.projectId, input.id);
|
|
23557
|
+
if (run.status !== "running" || run.execution?.token !== input.executionToken) {
|
|
23558
|
+
throw table.staleExecution(input.id);
|
|
23559
|
+
}
|
|
23560
|
+
return run;
|
|
23561
|
+
}
|
|
23562
|
+
async function confirmPgRunExecution(tx, table, input) {
|
|
23563
|
+
const existing = await lockHeldPgRun(tx, table, input);
|
|
23564
|
+
const [updated] = await tx.unsafe(`UPDATE ${table.name} SET execution_queue_lease_expires_at = $1
|
|
23565
|
+
WHERE project_id = $2 AND id = $3
|
|
23566
|
+
RETURNING ${table.returning}`, [
|
|
23567
|
+
laterLease(existing.execution?.queueLeaseExpiresAt, input.queueLeaseExpiresAt),
|
|
23568
|
+
input.projectId,
|
|
23569
|
+
input.id
|
|
23570
|
+
]);
|
|
23571
|
+
if (!updated)
|
|
23572
|
+
throw table.notFound(input.projectId, input.id);
|
|
23573
|
+
return table.toRun(updated);
|
|
23574
|
+
}
|
|
23575
|
+
async function listLostPgRuns(sql, table, input) {
|
|
23576
|
+
const rows = await sql.unsafe(`SELECT ${table.returning} FROM ${table.name}
|
|
23577
|
+
WHERE project_id = $1
|
|
23578
|
+
AND status IN ('queued', 'running')
|
|
23579
|
+
AND (
|
|
23580
|
+
(status = 'queued' AND queued_at <= $2)
|
|
23581
|
+
OR (status = 'running' AND execution_queue_lease_expires_at <= $3)
|
|
23582
|
+
OR (status = 'running' AND execution_queue_lease_expires_at IS NULL AND started_at <= $4)
|
|
23583
|
+
)
|
|
23584
|
+
ORDER BY status = 'running' DESC,
|
|
23585
|
+
COALESCE(execution_queue_lease_expires_at, started_at, queued_at)
|
|
23586
|
+
LIMIT $5`, [input.projectId, input.queuedBefore, input.lapsedBy, input.orphanedBefore, input.limit]);
|
|
23587
|
+
return rows.map(table.toRun);
|
|
23588
|
+
}
|
|
23589
|
+
async function releasePgRun(sql, table, input) {
|
|
23590
|
+
await sql.unsafe(`UPDATE ${table.name}
|
|
23591
|
+
SET execution_queue_lease_expires_at = $1, attempt = attempt - $2
|
|
23592
|
+
WHERE project_id = $3 AND id = $4 AND status = 'running' AND execution_token = $5`, [
|
|
23593
|
+
input.releasedAt ?? new Date,
|
|
23594
|
+
input.spentAttempt ? 0 : 1,
|
|
23595
|
+
input.projectId,
|
|
23596
|
+
input.id,
|
|
23597
|
+
input.executionToken
|
|
23598
|
+
]);
|
|
23599
|
+
}
|
|
23600
|
+
|
|
21756
23601
|
// src/pg-pipeline-run-storage.ts
|
|
23602
|
+
var PIPELINE_RUNS = {
|
|
23603
|
+
name: "pipeline_runs",
|
|
23604
|
+
returning: "*",
|
|
23605
|
+
toRun: rowToPipelineRunRecord,
|
|
23606
|
+
notFound: (projectId, id) => new PipelineRunError(`[SixbPg] Pipeline run '${id}' not found for project '${projectId}'.`),
|
|
23607
|
+
staleExecution
|
|
23608
|
+
};
|
|
23609
|
+
|
|
21757
23610
|
class PgPipelineRunStorage {
|
|
21758
23611
|
sql;
|
|
21759
23612
|
executions;
|
|
@@ -21811,6 +23664,7 @@ class PgPipelineRunStorage {
|
|
|
21811
23664
|
const [updated] = await tx`
|
|
21812
23665
|
UPDATE pipeline_runs
|
|
21813
23666
|
SET status = ${"queued"}, queued_at = ${input.queuedAt ?? new Date},
|
|
23667
|
+
attempt = ${0}, execution_token = ${null}, execution_queue_lease_expires_at = ${null},
|
|
21814
23668
|
started_at = ${null}, finished_at = ${null}, output_dataset_id = ${null},
|
|
21815
23669
|
output_version_id = ${null}, error = ${null}
|
|
21816
23670
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
@@ -21819,38 +23673,32 @@ class PgPipelineRunStorage {
|
|
|
21819
23673
|
return rowToPipelineRunRecord(updated);
|
|
21820
23674
|
});
|
|
21821
23675
|
}
|
|
21822
|
-
async
|
|
21823
|
-
|
|
21824
|
-
|
|
21825
|
-
|
|
21826
|
-
|
|
21827
|
-
|
|
21828
|
-
|
|
21829
|
-
|
|
21830
|
-
|
|
21831
|
-
|
|
21832
|
-
|
|
21833
|
-
if (!existing) {
|
|
21834
|
-
throw new PipelineRunError(`[SixbPg] Pipeline run '${input.id}' not found for project '${input.projectId}'.`);
|
|
21835
|
-
}
|
|
21836
|
-
throw new PipelineRunError(`[SixbPg] Pipeline run '${input.id}' cannot start from status '${existing.status}'.`);
|
|
23676
|
+
async startOrReclaim(input) {
|
|
23677
|
+
return runPgTransaction(this.sql, (tx) => claimPgRun(tx, PIPELINE_RUNS, input));
|
|
23678
|
+
}
|
|
23679
|
+
async confirmExecutionOwnership(input) {
|
|
23680
|
+
return runPgTransaction(this.sql, (tx) => confirmPgRunExecution(tx, PIPELINE_RUNS, input));
|
|
23681
|
+
}
|
|
23682
|
+
async listLost(input) {
|
|
23683
|
+
return listLostPgRuns(this.sql, PIPELINE_RUNS, input);
|
|
23684
|
+
}
|
|
23685
|
+
async releaseExecution(input) {
|
|
23686
|
+
await releasePgRun(this.sql, PIPELINE_RUNS, input);
|
|
21837
23687
|
}
|
|
21838
23688
|
async finish(input) {
|
|
21839
23689
|
return runPgTransaction(this.sql, async (tx) => {
|
|
21840
|
-
const
|
|
21841
|
-
SELECT * FROM pipeline_runs
|
|
21842
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
21843
|
-
`;
|
|
21844
|
-
if (!existing) {
|
|
21845
|
-
throw new PipelineRunError(`[SixbPg] Pipeline run '${input.id}' not found for project '${input.projectId}'.`);
|
|
21846
|
-
}
|
|
23690
|
+
const existing = await lockPgRun(tx, PIPELINE_RUNS, input.projectId, input.id);
|
|
21847
23691
|
if (!canFinishPipelineRun(existing.status, input)) {
|
|
21848
23692
|
throw new PipelineRunError(`[SixbPg] Pipeline run '${input.id}' cannot finish from status '${existing.status}'.`);
|
|
21849
23693
|
}
|
|
23694
|
+
if (!mayFinishRun(existing, input))
|
|
23695
|
+
throw staleExecution(input.id);
|
|
21850
23696
|
const [updated] = input.status === "succeeded" ? await tx`
|
|
21851
23697
|
UPDATE pipeline_runs
|
|
21852
23698
|
SET
|
|
21853
23699
|
status = ${input.status},
|
|
23700
|
+
execution_token = ${null},
|
|
23701
|
+
execution_queue_lease_expires_at = ${null},
|
|
21854
23702
|
finished_at = ${input.finishedAt ?? new Date},
|
|
21855
23703
|
output_dataset_id = ${input.output?.datasetId ?? null},
|
|
21856
23704
|
output_version_id = ${input.output?.versionId ?? null},
|
|
@@ -21861,6 +23709,8 @@ class PgPipelineRunStorage {
|
|
|
21861
23709
|
UPDATE pipeline_runs
|
|
21862
23710
|
SET
|
|
21863
23711
|
status = ${input.status},
|
|
23712
|
+
execution_token = ${null},
|
|
23713
|
+
execution_queue_lease_expires_at = ${null},
|
|
21864
23714
|
finished_at = ${input.finishedAt ?? new Date},
|
|
21865
23715
|
output_dataset_id = ${null},
|
|
21866
23716
|
output_version_id = ${null},
|
|
@@ -21873,19 +23723,23 @@ class PgPipelineRunStorage {
|
|
|
21873
23723
|
}
|
|
21874
23724
|
async startStep(input) {
|
|
21875
23725
|
return runPgTransaction(this.sql, async (tx) => {
|
|
21876
|
-
const
|
|
21877
|
-
SELECT * FROM pipeline_runs
|
|
21878
|
-
WHERE project_id = ${input.projectId} AND id = ${input.pipelineRunId}
|
|
21879
|
-
`;
|
|
21880
|
-
if (!pipelineRun) {
|
|
21881
|
-
throw new PipelineRunError(`[SixbPg] Pipeline run '${input.pipelineRunId}' not found for project '${input.projectId}'.`);
|
|
21882
|
-
}
|
|
23726
|
+
const pipelineRun = await lockPgRun(tx, PIPELINE_RUNS, input.projectId, input.pipelineRunId);
|
|
21883
23727
|
if (pipelineRun.status !== "running") {
|
|
21884
23728
|
throw new PipelineRunError(`[SixbPg] Pipeline run '${input.pipelineRunId}' for project '${input.projectId}' is already terminal.`);
|
|
21885
23729
|
}
|
|
21886
|
-
if (pipelineRun
|
|
21887
|
-
throw
|
|
23730
|
+
if (!holdsRun(pipelineRun, input.executionToken))
|
|
23731
|
+
throw staleExecution(input.pipelineRunId);
|
|
23732
|
+
if (pipelineRun.pipelineId !== input.pipelineId) {
|
|
23733
|
+
throw new PipelineRunError(`[SixbPg] Pipeline step run '${input.id}' pipeline '${input.pipelineId}' does not match pipeline run '${input.pipelineRunId}' pipeline '${pipelineRun.pipelineId}'.`);
|
|
21888
23734
|
}
|
|
23735
|
+
const [restarted] = await tx`
|
|
23736
|
+
UPDATE pipeline_step_runs
|
|
23737
|
+
SET inputs = ${JSON.stringify(input.inputs)}::text::jsonb
|
|
23738
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"running"}
|
|
23739
|
+
RETURNING *
|
|
23740
|
+
`;
|
|
23741
|
+
if (restarted)
|
|
23742
|
+
return rowToPipelineStepRunRecord(restarted);
|
|
21889
23743
|
try {
|
|
21890
23744
|
const [row] = await tx`
|
|
21891
23745
|
INSERT INTO pipeline_step_runs (
|
|
@@ -21935,6 +23789,10 @@ class PgPipelineRunStorage {
|
|
|
21935
23789
|
if (existing.status !== "running") {
|
|
21936
23790
|
throw new PipelineRunError(`[SixbPg] Pipeline step run '${input.id}' for project '${input.projectId}' is already terminal.`);
|
|
21937
23791
|
}
|
|
23792
|
+
const pipelineRun = await lockPgRun(tx, PIPELINE_RUNS, input.projectId, existing.pipeline_run_id);
|
|
23793
|
+
if (!holdsRun(pipelineRun, input.executionToken)) {
|
|
23794
|
+
throw staleExecution(existing.pipeline_run_id);
|
|
23795
|
+
}
|
|
21938
23796
|
if (input.status === "succeeded" && input.output.datasetId !== existing.dataset_id) {
|
|
21939
23797
|
throw new PipelineRunError(`[SixbPg] Pipeline step run '${input.id}' output dataset '${input.output.datasetId}' does not match '${existing.dataset_id}'.`);
|
|
21940
23798
|
}
|
|
@@ -22014,7 +23872,8 @@ class PgPipelineRunStorage {
|
|
|
22014
23872
|
ownerColumn: "pipeline_id",
|
|
22015
23873
|
ownerIds: input.pipelineIds,
|
|
22016
23874
|
projectId: input.projectId,
|
|
22017
|
-
ownerIdFor: (row) => row.pipeline_id
|
|
23875
|
+
ownerIdFor: (row) => row.pipeline_id,
|
|
23876
|
+
excludedStatus: input.excludedStatus
|
|
22018
23877
|
});
|
|
22019
23878
|
return {
|
|
22020
23879
|
runs: rows.map(rowToPipelineRunRecord)
|
|
@@ -22035,6 +23894,11 @@ class PgPipelineRunStorage {
|
|
|
22035
23894
|
whereClauses.push(`pipeline_run_id = $${index++}`);
|
|
22036
23895
|
params.push(input.pipelineRunId);
|
|
22037
23896
|
}
|
|
23897
|
+
if (input.pipelineRunIds) {
|
|
23898
|
+
const placeholders = input.pipelineRunIds.map(() => `$${index++}`);
|
|
23899
|
+
whereClauses.push(`pipeline_run_id IN (${placeholders.join(", ") || "NULL"})`);
|
|
23900
|
+
params.push(...input.pipelineRunIds);
|
|
23901
|
+
}
|
|
22038
23902
|
if (input.pipelineId) {
|
|
22039
23903
|
whereClauses.push(`pipeline_id = $${index++}`);
|
|
22040
23904
|
params.push(input.pipelineId);
|
|
@@ -22066,6 +23930,9 @@ class PgPipelineRunStorage {
|
|
|
22066
23930
|
};
|
|
22067
23931
|
}
|
|
22068
23932
|
}
|
|
23933
|
+
function staleExecution(id) {
|
|
23934
|
+
return new PipelineRunError(`[SixbPg] Execution token is no longer current on pipeline run '${id}'.`);
|
|
23935
|
+
}
|
|
22069
23936
|
function rowToPipelineRunRecord(row) {
|
|
22070
23937
|
return {
|
|
22071
23938
|
id: row.id,
|
|
@@ -22073,6 +23940,7 @@ function rowToPipelineRunRecord(row) {
|
|
|
22073
23940
|
executionId: row.execution_id,
|
|
22074
23941
|
pipelineId: row.pipeline_id,
|
|
22075
23942
|
status: row.status,
|
|
23943
|
+
...pgRunExecutionFields(row),
|
|
22076
23944
|
queuedAt: new Date(row.queued_at),
|
|
22077
23945
|
startedAt: row.started_at ? new Date(row.started_at) : undefined,
|
|
22078
23946
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
@@ -22143,6 +24011,7 @@ import {
|
|
|
22143
24011
|
SUPERSEDING_PROJECTION_RUN_STATUSES,
|
|
22144
24012
|
staleProjectionRunExecution
|
|
22145
24013
|
} from "@sixb/core/internal/projection-run-storage-provider";
|
|
24014
|
+
import { laterLease as laterLease2 } from "@sixb/core/internal/storage";
|
|
22146
24015
|
import { PROJECTION_RUN_FAILURE_CODES, ProjectionRunError } from "@sixb/core/storage";
|
|
22147
24016
|
class PgProjectionRunStorage {
|
|
22148
24017
|
sql;
|
|
@@ -22238,11 +24107,12 @@ class PgProjectionRunStorage {
|
|
|
22238
24107
|
throw projectionRunNotFound(input.projectId, input.id);
|
|
22239
24108
|
const existing = restoreProjectionRunRow(existingRow);
|
|
22240
24109
|
const { attempt } = planProjectionRunReclaim(existing, input);
|
|
22241
|
-
const executionToken = createFreshExecutionToken(existing.executionToken);
|
|
24110
|
+
const executionToken = input.execution?.token ?? createFreshExecutionToken(existing.executionToken);
|
|
22242
24111
|
const [updated] = await tx`
|
|
22243
24112
|
UPDATE projection_runs
|
|
22244
24113
|
SET status = ${"running"}, started_at = COALESCE(started_at, ${input.startedAt ?? new Date}),
|
|
22245
|
-
finished_at = ${null}, attempt = ${attempt}, execution_token = ${executionToken}
|
|
24114
|
+
finished_at = ${null}, attempt = ${attempt}, execution_token = ${executionToken},
|
|
24115
|
+
execution_queue_lease_expires_at = ${input.execution?.queueLeaseExpiresAt ?? null}
|
|
22246
24116
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
22247
24117
|
AND (
|
|
22248
24118
|
(status = ${"queued"} AND execution_token IS NULL)
|
|
@@ -22255,6 +24125,45 @@ class PgProjectionRunStorage {
|
|
|
22255
24125
|
return projectionRunClaim(updated);
|
|
22256
24126
|
});
|
|
22257
24127
|
}
|
|
24128
|
+
async confirmExecutionOwnership(input) {
|
|
24129
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
24130
|
+
const [existingRow] = await tx`
|
|
24131
|
+
SELECT * FROM projection_runs
|
|
24132
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
24133
|
+
FOR UPDATE
|
|
24134
|
+
`;
|
|
24135
|
+
if (!existingRow)
|
|
24136
|
+
throw projectionRunNotFound(input.projectId, input.id);
|
|
24137
|
+
const existing = restoreProjectionRunRow(existingRow);
|
|
24138
|
+
if (existing.status !== "running" || existing.executionToken !== input.executionToken) {
|
|
24139
|
+
throw staleProjectionRunExecution(input.id);
|
|
24140
|
+
}
|
|
24141
|
+
const [updated] = await tx`
|
|
24142
|
+
UPDATE projection_runs
|
|
24143
|
+
SET execution_queue_lease_expires_at = ${laterLease2(existing.queueLeaseExpiresAt, input.queueLeaseExpiresAt)}
|
|
24144
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
24145
|
+
RETURNING *
|
|
24146
|
+
`;
|
|
24147
|
+
return rowToProjectionRunRecord(updated);
|
|
24148
|
+
});
|
|
24149
|
+
}
|
|
24150
|
+
async listLapsed(input) {
|
|
24151
|
+
const rows = await this.sql`
|
|
24152
|
+
SELECT * FROM projection_runs
|
|
24153
|
+
WHERE project_id = ${input.projectId} AND status = ${"running"}
|
|
24154
|
+
AND execution_queue_lease_expires_at <= ${input.lapsedBy}
|
|
24155
|
+
ORDER BY execution_queue_lease_expires_at
|
|
24156
|
+
LIMIT ${input.limit}
|
|
24157
|
+
`;
|
|
24158
|
+
return rows.map(rowToProjectionRunRecord);
|
|
24159
|
+
}
|
|
24160
|
+
async releaseExecution(input) {
|
|
24161
|
+
await this.sql`
|
|
24162
|
+
UPDATE projection_runs SET execution_queue_lease_expires_at = ${input.releasedAt ?? new Date}
|
|
24163
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"running"}
|
|
24164
|
+
AND execution_token = ${input.executionToken}
|
|
24165
|
+
`;
|
|
24166
|
+
}
|
|
22258
24167
|
async failEnqueue(input) {
|
|
22259
24168
|
return runPgTransaction(this.sql, async (tx) => {
|
|
22260
24169
|
const [existingRow] = await tx`
|
|
@@ -22338,6 +24247,7 @@ class PgProjectionRunStorage {
|
|
|
22338
24247
|
status = ${input.status},
|
|
22339
24248
|
finished_at = ${plan.finishedAt},
|
|
22340
24249
|
execution_token = ${null},
|
|
24250
|
+
execution_queue_lease_expires_at = ${null},
|
|
22341
24251
|
source_rows_read = ${plan.progress.sourceRowsRead},
|
|
22342
24252
|
source_rows_skipped = ${plan.progress.sourceRowsSkipped},
|
|
22343
24253
|
source_changes_read = ${plan.progress.sourceChangesRead ?? null},
|
|
@@ -22601,6 +24511,7 @@ function restoreProjectionRunRow(row) {
|
|
|
22601
24511
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
22602
24512
|
attempt: databaseSafeInteger2(row.attempt, "attempt"),
|
|
22603
24513
|
executionToken: row.execution_token ?? undefined,
|
|
24514
|
+
queueLeaseExpiresAt: row.execution_queue_lease_expires_at ? new Date(row.execution_queue_lease_expires_at) : undefined,
|
|
22604
24515
|
fixedBatchSize: optionalDatabaseSafeInteger(row.fixed_batch_size, "fixedBatchSize"),
|
|
22605
24516
|
nextBatchOrdinal: optionalDatabaseSafeInteger(row.next_batch_ordinal, "nextBatchOrdinal"),
|
|
22606
24517
|
nextRowOffset: optionalDatabaseSafeInteger(row.next_row_offset, "nextRowOffset"),
|
|
@@ -22875,9 +24786,21 @@ function toIsoString3(value) {
|
|
|
22875
24786
|
|
|
22876
24787
|
// src/pg-sync-run-storage.ts
|
|
22877
24788
|
import { parseSixbFailure as parseSixbFailure5, serializeSixbFailure as serializeSixbFailure5 } from "@sixb/core/internal/errors";
|
|
22878
|
-
import {
|
|
24789
|
+
import {
|
|
24790
|
+
canFinishSyncRun,
|
|
24791
|
+
canRequeueSyncRunAfterEnqueueFailure,
|
|
24792
|
+
mayFinishRun as mayFinishRun2
|
|
24793
|
+
} from "@sixb/core/internal/storage";
|
|
22879
24794
|
import { assertSyncRunExecution } from "@sixb/core/internal/sync-run-storage-provider";
|
|
22880
24795
|
import { SYNC_RUN_FAILURE_CODES, SyncRunError } from "@sixb/core/storage";
|
|
24796
|
+
var SYNC_RUNS = {
|
|
24797
|
+
name: "sync_runs",
|
|
24798
|
+
returning: "*, checkpoint IS NOT NULL AS checkpoint_present",
|
|
24799
|
+
toRun: rowToSyncRunRecord,
|
|
24800
|
+
notFound: (projectId, id) => new SyncRunError(`[SixbPg] Sync run '${id}' not found for project '${projectId}'.`),
|
|
24801
|
+
staleExecution: staleExecution2
|
|
24802
|
+
};
|
|
24803
|
+
|
|
22881
24804
|
class PgSyncRunStorage {
|
|
22882
24805
|
sql;
|
|
22883
24806
|
executions;
|
|
@@ -22943,7 +24866,9 @@ class PgSyncRunStorage {
|
|
|
22943
24866
|
const [updated] = await tx`
|
|
22944
24867
|
UPDATE sync_runs
|
|
22945
24868
|
SET status = ${"queued"}, queued_at = ${input.queuedAt ?? new Date},
|
|
22946
|
-
|
|
24869
|
+
attempt = ${0}, execution_token = ${null}, execution_queue_lease_expires_at = ${null},
|
|
24870
|
+
commit_prepared_at = ${null}, started_at = ${null}, finished_at = ${null},
|
|
24871
|
+
rows_read = ${null},
|
|
22947
24872
|
output_version_id = ${null}, error = ${null}, checkpoint = ${null}
|
|
22948
24873
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
22949
24874
|
RETURNING *, checkpoint IS NOT NULL AS checkpoint_present
|
|
@@ -22951,39 +24876,55 @@ class PgSyncRunStorage {
|
|
|
22951
24876
|
return rowToSyncRunRecord(updated);
|
|
22952
24877
|
});
|
|
22953
24878
|
}
|
|
22954
|
-
async
|
|
22955
|
-
const [updated] = await this.sql`
|
|
22956
|
-
UPDATE sync_runs
|
|
22957
|
-
SET status = ${"running"}, started_at = ${input.startedAt ?? new Date},
|
|
22958
|
-
error = ${null}
|
|
22959
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"queued"}
|
|
22960
|
-
RETURNING *, checkpoint IS NOT NULL AS checkpoint_present
|
|
22961
|
-
`;
|
|
22962
|
-
if (updated)
|
|
22963
|
-
return rowToSyncRunRecord(updated);
|
|
22964
|
-
const existing = await this.getById({ projectId: input.projectId, id: input.id });
|
|
22965
|
-
if (!existing) {
|
|
22966
|
-
throw new SyncRunError(`[SixbPg] Sync run '${input.id}' not found for project '${input.projectId}'.`);
|
|
22967
|
-
}
|
|
22968
|
-
throw new SyncRunError(`[SixbPg] Sync run '${input.id}' cannot start from status '${existing.status}'.`);
|
|
22969
|
-
}
|
|
22970
|
-
async finish(input) {
|
|
24879
|
+
async startOrReclaim(input) {
|
|
22971
24880
|
return runPgTransaction(this.sql, async (tx) => {
|
|
22972
|
-
const
|
|
24881
|
+
const { syncId } = await lockPgRun(tx, SYNC_RUNS, input.projectId, input.id);
|
|
24882
|
+
await lockAdvisoryKeys(tx, [syncRunsLockKey(input.projectId, syncId)]);
|
|
24883
|
+
const [sibling] = await tx`
|
|
22973
24884
|
SELECT *, checkpoint IS NOT NULL AS checkpoint_present FROM sync_runs
|
|
24885
|
+
WHERE project_id = ${input.projectId} AND sync_id = ${syncId}
|
|
24886
|
+
AND status = ${"running"} AND id <> ${input.id}
|
|
24887
|
+
LIMIT 1
|
|
24888
|
+
`;
|
|
24889
|
+
return claimPgRun(tx, SYNC_RUNS, input, sibling && rowToSyncRunRecord(sibling));
|
|
24890
|
+
});
|
|
24891
|
+
}
|
|
24892
|
+
async confirmExecutionOwnership(input) {
|
|
24893
|
+
return runPgTransaction(this.sql, (tx) => confirmPgRunExecution(tx, SYNC_RUNS, input));
|
|
24894
|
+
}
|
|
24895
|
+
async listLost(input) {
|
|
24896
|
+
return listLostPgRuns(this.sql, SYNC_RUNS, input);
|
|
24897
|
+
}
|
|
24898
|
+
async releaseExecution(input) {
|
|
24899
|
+
await releasePgRun(this.sql, SYNC_RUNS, input);
|
|
24900
|
+
}
|
|
24901
|
+
async prepareCommit(input) {
|
|
24902
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
24903
|
+
await lockHeldPgRun(tx, SYNC_RUNS, input);
|
|
24904
|
+
const [updated] = await tx`
|
|
24905
|
+
UPDATE sync_runs
|
|
24906
|
+
SET rows_read = ${input.rowsRead},
|
|
24907
|
+
checkpoint = ${serializeCheckpoint(input.checkpoint)}::text::jsonb,
|
|
24908
|
+
commit_prepared_at = ${input.preparedAt ?? new Date}
|
|
22974
24909
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
24910
|
+
RETURNING *, checkpoint IS NOT NULL AS checkpoint_present
|
|
22975
24911
|
`;
|
|
22976
|
-
|
|
22977
|
-
|
|
22978
|
-
|
|
22979
|
-
|
|
22980
|
-
|
|
24912
|
+
return rowToSyncRunRecord(updated);
|
|
24913
|
+
});
|
|
24914
|
+
}
|
|
24915
|
+
async finish(input) {
|
|
24916
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
24917
|
+
const record = await lockPgRun(tx, SYNC_RUNS, input.projectId, input.id);
|
|
24918
|
+
if (!canFinishSyncRun(record.status, input)) {
|
|
24919
|
+
throw new SyncRunError(`[SixbPg] Sync run '${input.id}' cannot finish from status '${record.status}'.`);
|
|
22981
24920
|
}
|
|
24921
|
+
if (!mayFinishRun2(record, input))
|
|
24922
|
+
throw staleExecution2(input.id);
|
|
22982
24923
|
if (input.status === "succeeded") {
|
|
22983
|
-
if (input.output && input.output.datasetId !==
|
|
22984
|
-
throw new SyncRunError(`[SixbPg] Sync run '${input.id}' output dataset '${input.output.datasetId}' does not match '${
|
|
24924
|
+
if (input.output && input.output.datasetId !== record.datasetId) {
|
|
24925
|
+
throw new SyncRunError(`[SixbPg] Sync run '${input.id}' output dataset '${input.output.datasetId}' does not match '${record.datasetId}'.`);
|
|
22985
24926
|
}
|
|
22986
|
-
if (!input.output && input.rowsRead !== 0 &&
|
|
24927
|
+
if (!input.output && input.rowsRead !== 0 && record.mode !== "merge") {
|
|
22987
24928
|
throw new SyncRunError(`[SixbPg] Sync run '${input.id}' may omit its output with rows read only for an initial merge no-op.`);
|
|
22988
24929
|
}
|
|
22989
24930
|
}
|
|
@@ -22991,6 +24932,8 @@ class PgSyncRunStorage {
|
|
|
22991
24932
|
UPDATE sync_runs
|
|
22992
24933
|
SET
|
|
22993
24934
|
status = ${input.status},
|
|
24935
|
+
execution_token = ${null},
|
|
24936
|
+
execution_queue_lease_expires_at = ${null},
|
|
22994
24937
|
finished_at = ${input.finishedAt ?? new Date},
|
|
22995
24938
|
rows_read = ${input.rowsRead},
|
|
22996
24939
|
output_version_id = ${input.output?.versionId ?? null},
|
|
@@ -23002,8 +24945,10 @@ class PgSyncRunStorage {
|
|
|
23002
24945
|
UPDATE sync_runs
|
|
23003
24946
|
SET
|
|
23004
24947
|
status = ${input.status},
|
|
24948
|
+
execution_token = ${null},
|
|
24949
|
+
execution_queue_lease_expires_at = ${null},
|
|
23005
24950
|
finished_at = ${input.finishedAt ?? new Date},
|
|
23006
|
-
rows_read = ${input.rowsRead ??
|
|
24951
|
+
rows_read = ${input.rowsRead ?? record.rowsRead ?? null},
|
|
23007
24952
|
output_version_id = ${null},
|
|
23008
24953
|
error = ${input.error === undefined ? null : serializeSixbFailure5(input.error, SYNC_RUN_FAILURE_CODES)}::text::jsonb,
|
|
23009
24954
|
checkpoint = ${null}
|
|
@@ -23077,6 +25022,12 @@ class PgSyncRunStorage {
|
|
|
23077
25022
|
};
|
|
23078
25023
|
}
|
|
23079
25024
|
}
|
|
25025
|
+
function staleExecution2(id) {
|
|
25026
|
+
return new SyncRunError(`[SixbPg] Execution token is no longer current on sync run '${id}'.`);
|
|
25027
|
+
}
|
|
25028
|
+
function syncRunsLockKey(projectId, syncId) {
|
|
25029
|
+
return `sync-runs:${JSON.stringify([projectId, syncId])}`;
|
|
25030
|
+
}
|
|
23080
25031
|
function serializeCheckpoint(checkpoint) {
|
|
23081
25032
|
return checkpoint !== undefined ? JSON.stringify(checkpoint) : null;
|
|
23082
25033
|
}
|
|
@@ -23089,6 +25040,8 @@ function rowToSyncRunRecord(row) {
|
|
|
23089
25040
|
datasetId: row.dataset_id,
|
|
23090
25041
|
mode: row.mode,
|
|
23091
25042
|
status: row.status,
|
|
25043
|
+
...pgRunExecutionFields(row),
|
|
25044
|
+
commitPreparedAt: row.commit_prepared_at ? new Date(row.commit_prepared_at) : undefined,
|
|
23092
25045
|
queuedAt: new Date(row.queued_at),
|
|
23093
25046
|
startedAt: row.started_at ? new Date(row.started_at) : undefined,
|
|
23094
25047
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
@@ -23246,10 +25199,12 @@ class PgWebhookRunStorage {
|
|
|
23246
25199
|
const [row] = await this.sql`
|
|
23247
25200
|
INSERT INTO webhook_runs (
|
|
23248
25201
|
project_id, id, execution_id, connector_id, webhook_id, status, method, route,
|
|
25202
|
+
execution_token, execution_lease_expires_at,
|
|
23249
25203
|
started_at, request_body_bytes, request_body_sha256, idempotency_key
|
|
23250
25204
|
) VALUES (
|
|
23251
25205
|
${input.projectId}, ${input.id}, ${input.executionId}, ${input.connectorId},
|
|
23252
25206
|
${input.webhookId}, ${"running"}, ${input.method}, ${input.route},
|
|
25207
|
+
${input.execution.token}, ${input.execution.leaseExpiresAt},
|
|
23253
25208
|
${input.startedAt ?? new Date}, ${input.requestBodyBytes},
|
|
23254
25209
|
${input.requestBodySha256}, ${input.idempotencyKey ?? null}
|
|
23255
25210
|
)
|
|
@@ -23273,22 +25228,31 @@ class PgWebhookRunStorage {
|
|
|
23273
25228
|
if (!existing) {
|
|
23274
25229
|
throw new WebhookRunError(`[SixbPg] Webhook run '${input.id}' not found for project '${input.projectId}'.`, "not_found");
|
|
23275
25230
|
}
|
|
23276
|
-
|
|
25231
|
+
const startedAt = input.startedAt ?? new Date;
|
|
25232
|
+
if (!canRetryWebhookRun(rowToWebhookRunRecord(existing), startedAt)) {
|
|
23277
25233
|
throw new WebhookRunError(`[SixbPg] Webhook run '${input.id}' cannot restart from status '${existing.status}'.`, "invalid_transition");
|
|
23278
25234
|
}
|
|
23279
25235
|
const [updated] = await tx`
|
|
23280
25236
|
UPDATE webhook_runs
|
|
23281
|
-
SET status = ${"running"},
|
|
23282
|
-
|
|
23283
|
-
|
|
25237
|
+
SET status = ${"running"}, execution_token = ${input.execution.token},
|
|
25238
|
+
execution_lease_expires_at = ${input.execution.leaseExpiresAt},
|
|
25239
|
+
started_at = ${startedAt}, finished_at = ${null}, response_status = ${null},
|
|
25240
|
+
error = ${null}
|
|
25241
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
23284
25242
|
RETURNING *
|
|
23285
25243
|
`;
|
|
23286
|
-
if (!updated) {
|
|
23287
|
-
throw new WebhookRunError(`[SixbPg] Webhook run '${input.id}' changed while restarting.`, "invalid_transition");
|
|
23288
|
-
}
|
|
23289
25244
|
return rowToWebhookRunRecord(updated);
|
|
23290
25245
|
});
|
|
23291
25246
|
}
|
|
25247
|
+
async renewLease(input) {
|
|
25248
|
+
const updated = await this.sql`
|
|
25249
|
+
UPDATE webhook_runs SET execution_lease_expires_at = ${input.leaseExpiresAt}
|
|
25250
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"running"}
|
|
25251
|
+
AND execution_token = ${input.executionToken}
|
|
25252
|
+
RETURNING *
|
|
25253
|
+
`;
|
|
25254
|
+
return updated.length > 0;
|
|
25255
|
+
}
|
|
23292
25256
|
async finish(input) {
|
|
23293
25257
|
return runPgTransaction(this.sql, async (tx) => {
|
|
23294
25258
|
const [existing] = await tx`
|
|
@@ -23302,9 +25266,13 @@ class PgWebhookRunStorage {
|
|
|
23302
25266
|
if (existing.status !== "running") {
|
|
23303
25267
|
throw new WebhookRunError(`[SixbPg] Webhook run '${input.id}' for project '${input.projectId}' is already terminal.`, "invalid_transition");
|
|
23304
25268
|
}
|
|
25269
|
+
if (existing.execution_token !== input.executionToken) {
|
|
25270
|
+
throw new WebhookRunError(`[SixbPg] Execution token is no longer current on webhook run '${input.id}'.`, "invalid_transition");
|
|
25271
|
+
}
|
|
23305
25272
|
const [updated] = await tx`
|
|
23306
25273
|
UPDATE webhook_runs
|
|
23307
|
-
SET status = ${input.status},
|
|
25274
|
+
SET status = ${input.status}, execution_token = ${null},
|
|
25275
|
+
execution_lease_expires_at = ${null}, finished_at = ${input.finishedAt ?? new Date},
|
|
23308
25276
|
response_status = ${input.responseStatus ?? null},
|
|
23309
25277
|
error = ${input.status === "succeeded" ? null : serializeSixbFailure6(input.error, WEBHOOK_RUN_FAILURE_CODES)}::text::jsonb
|
|
23310
25278
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
@@ -23370,6 +25338,10 @@ function rowToWebhookRunRecord(row) {
|
|
|
23370
25338
|
connectorId: row.connector_id,
|
|
23371
25339
|
webhookId: row.webhook_id,
|
|
23372
25340
|
status: row.status,
|
|
25341
|
+
execution: row.execution_token && row.execution_lease_expires_at ? {
|
|
25342
|
+
token: row.execution_token,
|
|
25343
|
+
leaseExpiresAt: new Date(row.execution_lease_expires_at)
|
|
25344
|
+
} : undefined,
|
|
23373
25345
|
method: row.method,
|
|
23374
25346
|
route: row.route,
|
|
23375
25347
|
startedAt: new Date(row.started_at),
|
|
@@ -23623,6 +25595,7 @@ function assertNonNegativeInteger(value, fieldName) {
|
|
|
23623
25595
|
|
|
23624
25596
|
// src/pg-workflow-run-storage.ts
|
|
23625
25597
|
import { parseSixbFailure as parseSixbFailure7, serializeSixbFailure as serializeSixbFailure7 } from "@sixb/core/internal/errors";
|
|
25598
|
+
import { holdsRun as holdsRun2, mayFinishRun as mayFinishRun3 } from "@sixb/core/internal/storage";
|
|
23626
25599
|
import {
|
|
23627
25600
|
assertWorkflowAgentNodeRunExecution,
|
|
23628
25601
|
assertWorkflowRunExecution
|
|
@@ -23632,6 +25605,14 @@ import {
|
|
|
23632
25605
|
WORKFLOW_RUN_FAILURE_CODES,
|
|
23633
25606
|
WorkflowRunError
|
|
23634
25607
|
} from "@sixb/core/storage";
|
|
25608
|
+
var WORKFLOW_RUNS = {
|
|
25609
|
+
name: "workflow_runs",
|
|
25610
|
+
returning: "*",
|
|
25611
|
+
toRun: rowToWorkflowRunRecord,
|
|
25612
|
+
notFound: (projectId, id) => new WorkflowRunError(`[SixbPg] Workflow run '${id}' not found for project '${projectId}'.`),
|
|
25613
|
+
staleExecution: (id) => new WorkflowRunError(`[SixbPg] Execution token is no longer current on workflow run '${id}'.`)
|
|
25614
|
+
};
|
|
25615
|
+
|
|
23635
25616
|
class PgWorkflowRunStorage {
|
|
23636
25617
|
sql;
|
|
23637
25618
|
executions;
|
|
@@ -23685,59 +25666,17 @@ class PgWorkflowRunStorage {
|
|
|
23685
25666
|
throw error;
|
|
23686
25667
|
}
|
|
23687
25668
|
}
|
|
23688
|
-
async
|
|
23689
|
-
return runPgTransaction(this.sql,
|
|
23690
|
-
const startedAt = input.startedAt ?? new Date;
|
|
23691
|
-
const [existing] = await tx`
|
|
23692
|
-
SELECT * FROM workflow_runs
|
|
23693
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
23694
|
-
FOR UPDATE
|
|
23695
|
-
`;
|
|
23696
|
-
if (!existing || existing.status !== "queued") {
|
|
23697
|
-
throw new WorkflowRunError(existing ? `[SixbPg] Workflow run '${input.id}' cannot start from status '${existing.status}'.` : `[SixbPg] Workflow run '${input.id}' not found for project '${input.projectId}'.`);
|
|
23698
|
-
}
|
|
23699
|
-
const [updated] = await tx`
|
|
23700
|
-
UPDATE workflow_runs
|
|
23701
|
-
SET
|
|
23702
|
-
status = ${"running"},
|
|
23703
|
-
started_at = ${startedAt},
|
|
23704
|
-
finished_at = ${null},
|
|
23705
|
-
error = ${null},
|
|
23706
|
-
attempt = attempt + 1,
|
|
23707
|
-
execution_token = ${input.execution?.token ?? null},
|
|
23708
|
-
execution_queue_lease_expires_at = ${input.execution?.queueLeaseExpiresAt ?? null}
|
|
23709
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
23710
|
-
RETURNING *
|
|
23711
|
-
`;
|
|
23712
|
-
return rowToWorkflowRunRecord(updated);
|
|
23713
|
-
});
|
|
25669
|
+
async startOrReclaim(input) {
|
|
25670
|
+
return runPgTransaction(this.sql, (tx) => claimPgRun(tx, WORKFLOW_RUNS, input));
|
|
23714
25671
|
}
|
|
23715
|
-
async
|
|
23716
|
-
return
|
|
23717
|
-
|
|
23718
|
-
|
|
23719
|
-
|
|
23720
|
-
attempt = ${Number(existing.attempt) + 1},
|
|
23721
|
-
execution_token = ${input.execution.token},
|
|
23722
|
-
execution_queue_lease_expires_at = ${input.execution.queueLeaseExpiresAt}
|
|
23723
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
23724
|
-
RETURNING *
|
|
23725
|
-
`;
|
|
23726
|
-
return rowToWorkflowRunRecord(updated);
|
|
23727
|
-
});
|
|
25672
|
+
async listLost(input) {
|
|
25673
|
+
return listLostPgRuns(this.sql, WORKFLOW_RUNS, input);
|
|
25674
|
+
}
|
|
25675
|
+
async releaseExecution(input) {
|
|
25676
|
+
await releasePgRun(this.sql, WORKFLOW_RUNS, input);
|
|
23728
25677
|
}
|
|
23729
25678
|
async confirmExecutionOwnership(input) {
|
|
23730
|
-
return runPgTransaction(this.sql,
|
|
23731
|
-
const existing = await lockWorkflowRun(tx, input.projectId, input.id, "running");
|
|
23732
|
-
assertWorkflowRunExecutionOwnership(existing, input.executionToken);
|
|
23733
|
-
const current = existing.execution_queue_lease_expires_at ? new Date(existing.execution_queue_lease_expires_at) : input.queueLeaseExpiresAt;
|
|
23734
|
-
const [updated] = await tx`
|
|
23735
|
-
UPDATE workflow_runs SET execution_queue_lease_expires_at = ${new Date(Math.max(current.getTime(), input.queueLeaseExpiresAt.getTime()))}
|
|
23736
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
23737
|
-
RETURNING *
|
|
23738
|
-
`;
|
|
23739
|
-
return rowToWorkflowRunRecord(updated);
|
|
23740
|
-
});
|
|
25679
|
+
return runPgTransaction(this.sql, (tx) => confirmPgRunExecution(tx, WORKFLOW_RUNS, input));
|
|
23741
25680
|
}
|
|
23742
25681
|
async finish(input) {
|
|
23743
25682
|
return runPgTransaction(this.sql, async (tx) => {
|
|
@@ -23752,7 +25691,9 @@ class PgWorkflowRunStorage {
|
|
|
23752
25691
|
if (!canFinishWorkflowRun(existing.status, input.status)) {
|
|
23753
25692
|
throw new WorkflowRunError(`[SixbPg] Workflow run '${input.id}' for project '${input.projectId}' cannot be finished from status '${existing.status}'.`);
|
|
23754
25693
|
}
|
|
23755
|
-
|
|
25694
|
+
if (!mayFinishRun3(rowToWorkflowRunRecord(existing), input)) {
|
|
25695
|
+
throw WORKFLOW_RUNS.staleExecution(input.id);
|
|
25696
|
+
}
|
|
23756
25697
|
const [updated] = input.status === "succeeded" ? await tx`
|
|
23757
25698
|
UPDATE workflow_runs
|
|
23758
25699
|
SET
|
|
@@ -23826,7 +25767,7 @@ class PgWorkflowRunStorage {
|
|
|
23826
25767
|
status = ${"running"},
|
|
23827
25768
|
finished_at = ${null},
|
|
23828
25769
|
error = ${null},
|
|
23829
|
-
attempt =
|
|
25770
|
+
attempt = ${1},
|
|
23830
25771
|
execution_token = ${input.execution?.token ?? null},
|
|
23831
25772
|
execution_queue_lease_expires_at = ${input.execution?.queueLeaseExpiresAt ?? null}
|
|
23832
25773
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
@@ -24282,13 +26223,7 @@ function rowToWorkflowRunRecord(row) {
|
|
|
24282
26223
|
startedAt: new Date(row.started_at),
|
|
24283
26224
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
24284
26225
|
error: row.error === null ? undefined : parseSixbFailure7(row.error, WORKFLOW_RUN_FAILURE_CODES),
|
|
24285
|
-
|
|
24286
|
-
...row.execution_token && row.execution_queue_lease_expires_at ? {
|
|
24287
|
-
execution: {
|
|
24288
|
-
token: row.execution_token,
|
|
24289
|
-
queueLeaseExpiresAt: new Date(row.execution_queue_lease_expires_at)
|
|
24290
|
-
}
|
|
24291
|
-
} : {}
|
|
26226
|
+
...pgRunExecutionFields(row)
|
|
24292
26227
|
};
|
|
24293
26228
|
}
|
|
24294
26229
|
function rowToWorkflowNodeRunRecord(row) {
|
|
@@ -24364,21 +26299,9 @@ function rowToWorkflowAgentNodeRunRecord(row) {
|
|
|
24364
26299
|
function parseJson2(value) {
|
|
24365
26300
|
return typeof value === "string" ? JSON.parse(value) : value;
|
|
24366
26301
|
}
|
|
24367
|
-
async function lockWorkflowRun(sql, projectId, id, status) {
|
|
24368
|
-
const [row] = await sql`
|
|
24369
|
-
SELECT * FROM workflow_runs WHERE project_id = ${projectId} AND id = ${id} FOR UPDATE
|
|
24370
|
-
`;
|
|
24371
|
-
if (!row)
|
|
24372
|
-
throw new WorkflowRunError(`[SixbPg] Workflow run '${id}' not found.`);
|
|
24373
|
-
if (row.status !== status) {
|
|
24374
|
-
throw new WorkflowRunError(`[SixbPg] Workflow run '${id}' must be ${status} (status '${row.status}').`);
|
|
24375
|
-
}
|
|
24376
|
-
return row;
|
|
24377
|
-
}
|
|
24378
26302
|
function assertWorkflowRunExecutionOwnership(row, token) {
|
|
24379
|
-
if (row
|
|
24380
|
-
throw
|
|
24381
|
-
}
|
|
26303
|
+
if (!holdsRun2(rowToWorkflowRunRecord(row), token))
|
|
26304
|
+
throw WORKFLOW_RUNS.staleExecution(row.id);
|
|
24382
26305
|
}
|
|
24383
26306
|
async function assertWorkflowNodeParentExecutionOwnership(sql, node, token) {
|
|
24384
26307
|
const [run] = await sql`
|
|
@@ -24591,6 +26514,7 @@ import {
|
|
|
24591
26514
|
normalizeGetShareSessionByIdInput,
|
|
24592
26515
|
normalizeRenewShareSessionIfValidInput,
|
|
24593
26516
|
normalizeRevokeShareSessionInput,
|
|
26517
|
+
normalizeSetShareSessionGuestNameInput,
|
|
24594
26518
|
normalizeShareSessionCreate,
|
|
24595
26519
|
parseShareSessionRecord
|
|
24596
26520
|
} from "@sixb/core/internal/share-session-storage-provider";
|
|
@@ -24650,6 +26574,21 @@ class PgShareSessionStorage {
|
|
|
24650
26574
|
`;
|
|
24651
26575
|
return row ? recordFromRow2(row) : null;
|
|
24652
26576
|
}
|
|
26577
|
+
async setGuestName(input) {
|
|
26578
|
+
const normalized = normalizeSetShareSessionGuestNameInput(input);
|
|
26579
|
+
const [row] = await this.sql`
|
|
26580
|
+
UPDATE share_sessions
|
|
26581
|
+
SET guest_name = ${normalized.name}
|
|
26582
|
+
WHERE project_id = ${normalized.projectId}
|
|
26583
|
+
AND id = ${normalized.id}
|
|
26584
|
+
AND grant_id = ${normalized.grantId}
|
|
26585
|
+
AND revoked_at IS NULL
|
|
26586
|
+
AND expires_at > ${normalized.now}
|
|
26587
|
+
AND absolute_expires_at > ${normalized.now}
|
|
26588
|
+
RETURNING *
|
|
26589
|
+
`;
|
|
26590
|
+
return row ? recordFromRow2(row) : null;
|
|
26591
|
+
}
|
|
24653
26592
|
async revoke(input) {
|
|
24654
26593
|
const normalized = normalizeRevokeShareSessionInput(input);
|
|
24655
26594
|
const current = await this.getById(normalized);
|
|
@@ -24678,7 +26617,8 @@ function recordFromRow2(row) {
|
|
|
24678
26617
|
createdAt: row.created_at,
|
|
24679
26618
|
expiresAt: row.expires_at,
|
|
24680
26619
|
absoluteExpiresAt: row.absolute_expires_at,
|
|
24681
|
-
...row.revoked_at === null ? {} : { revokedAt: row.revoked_at }
|
|
26620
|
+
...row.revoked_at === null ? {} : { revokedAt: row.revoked_at },
|
|
26621
|
+
...row.guest_name === null ? {} : { guestName: row.guest_name }
|
|
24682
26622
|
});
|
|
24683
26623
|
}
|
|
24684
26624
|
function invalidRecord2(message, cause) {
|
|
@@ -24715,8 +26655,11 @@ class PostgresStorage {
|
|
|
24715
26655
|
shareGrants;
|
|
24716
26656
|
shareSessions;
|
|
24717
26657
|
fileUploadSessions;
|
|
26658
|
+
datasetIngest;
|
|
24718
26659
|
fileDownloadGrants;
|
|
24719
26660
|
activity;
|
|
26661
|
+
maintenanceHeartbeats;
|
|
26662
|
+
lakeRetentionRuns;
|
|
24720
26663
|
migrators;
|
|
24721
26664
|
sql;
|
|
24722
26665
|
schemaName;
|
|
@@ -24777,8 +26720,11 @@ class PostgresStorage {
|
|
|
24777
26720
|
this.shareGrants = createOperationScopedFacade(stores.shareGrants, scope);
|
|
24778
26721
|
this.shareSessions = createOperationScopedFacade(stores.shareSessions, scope);
|
|
24779
26722
|
this.fileUploadSessions = createOperationScopedFacade(stores.fileUploadSessions, scope);
|
|
26723
|
+
this.datasetIngest = createOperationScopedFacade(stores.datasetIngest, scope);
|
|
24780
26724
|
this.fileDownloadGrants = createOperationScopedFacade(stores.fileDownloadGrants, scope);
|
|
24781
26725
|
this.activity = createOperationScopedFacade(stores.activity, scope);
|
|
26726
|
+
this.maintenanceHeartbeats = createOperationScopedFacade(stores.maintenanceHeartbeats, scope);
|
|
26727
|
+
this.lakeRetentionRuns = createOperationScopedFacade(stores.lakeRetentionRuns, scope);
|
|
24782
26728
|
registerPostgresStorageTestingAdapter(this, (durationMs) => stores.connectorConnections.advanceTimeForTesting(durationMs));
|
|
24783
26729
|
}
|
|
24784
26730
|
async ping() {
|
|
@@ -24897,8 +26843,11 @@ function createPostgresStores(sql, options) {
|
|
|
24897
26843
|
shareGrants,
|
|
24898
26844
|
shareSessions,
|
|
24899
26845
|
fileUploadSessions: new PgFileUploadSessionStorage(sql),
|
|
26846
|
+
datasetIngest: new PgDatasetIngestStorage(sql),
|
|
24900
26847
|
fileDownloadGrants: new PgFileDownloadGrantStorage(sql),
|
|
24901
|
-
activity: new PgActivityStorage(sql)
|
|
26848
|
+
activity: new PgActivityStorage(sql),
|
|
26849
|
+
maintenanceHeartbeats: new PgMaintenanceHeartbeatStorage(sql),
|
|
26850
|
+
lakeRetentionRuns: new PgLakeRetentionRunStorage(sql)
|
|
24902
26851
|
};
|
|
24903
26852
|
}
|
|
24904
26853
|
function resolveTimeoutMillis(value, label) {
|
|
@@ -24915,4 +26864,4 @@ export {
|
|
|
24915
26864
|
migratePostgresStorage
|
|
24916
26865
|
};
|
|
24917
26866
|
|
|
24918
|
-
//# debugId=
|
|
26867
|
+
//# debugId=3787CB9C13986BA564756E2164756E21
|