@sixb/pg 0.1.25 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -1
- package/dist/agents/agent-storage.d.ts +8 -3
- package/dist/agents/agent-storage.d.ts.map +1 -1
- package/dist/agents/computers.d.ts +22 -0
- package/dist/agents/computers.d.ts.map +1 -0
- package/dist/agents/messages.d.ts.map +1 -1
- package/dist/agents/rows.d.ts +6 -0
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/runs.d.ts +2 -1
- package/dist/agents/runs.d.ts.map +1 -1
- package/dist/agents/threads.d.ts +3 -1
- package/dist/agents/threads.d.ts.map +1 -1
- package/dist/agents/token-rates.d.ts +13 -0
- package/dist/agents/token-rates.d.ts.map +1 -0
- package/dist/ai-accounting-filters.d.ts +12 -0
- package/dist/ai-accounting-filters.d.ts.map +1 -0
- package/dist/ai-cost-groups.d.ts +1 -1
- package/dist/ai-cost-groups.d.ts.map +1 -1
- package/dist/index.d.ts +5 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1446 -588
- package/dist/index.js.map +30 -25
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/index.d.ts +1 -0
- package/dist/objects/index.d.ts.map +1 -1
- package/dist/objects/partitions.d.ts +10 -0
- package/dist/objects/partitions.d.ts.map +1 -0
- package/dist/objects/prepare-queries.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +5 -0
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/query-indexes.d.ts +19 -1
- package/dist/objects/query-indexes.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +3 -1
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/sql-text.d.ts +3 -0
- package/dist/objects/sql-text.d.ts.map +1 -0
- package/dist/objects/storage.d.ts +3 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/objects/text-count-index.d.ts.map +1 -1
- package/dist/ontology-storage/commits.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +7 -1
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +2 -0
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +2 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/pg-ai-cost-storage.d.ts.map +1 -1
- package/dist/pg-client.d.ts +1 -1
- package/dist/pg-client.d.ts.map +1 -1
- package/dist/storage-errors.d.ts +4 -1
- package/dist/storage-errors.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/agent-storage.ts +10 -3
- package/src/agents/computers.ts +95 -0
- package/src/agents/messages.ts +15 -0
- package/src/agents/rows.ts +14 -0
- package/src/agents/runs.ts +56 -4
- package/src/agents/threads.ts +123 -2
- package/src/agents/token-rates.ts +76 -0
- package/src/ai-accounting-filters.ts +48 -0
- package/src/ai-cost-groups.ts +37 -12
- package/src/index.ts +21 -8
- package/src/migrations/082-agent-run-cancel-requests.sql +3 -0
- package/src/migrations/083-agent-thread-runs-on.sql +4 -0
- package/src/migrations/084-agent-token-rates.sql +16 -0
- package/src/migrations/085-agent-computer-sessions.sql +27 -0
- package/src/migrations/086-partition-objects.sql +93 -0
- package/src/migrations.ts +14 -0
- package/src/objects/index.ts +1 -0
- package/src/objects/partitions.ts +163 -0
- package/src/objects/prepare-queries.ts +50 -2
- package/src/objects/query-compiler.ts +136 -96
- package/src/objects/query-indexes.ts +120 -63
- package/src/objects/reader.ts +5 -4
- package/src/objects/sql-text.ts +8 -0
- package/src/objects/storage.ts +14 -2
- package/src/objects/text-count-index.ts +17 -12
- package/src/ontology-storage/commits.ts +11 -2
- package/src/ontology-storage/materialization-session.ts +10 -1
- package/src/ontology-storage/materialization-writer.ts +14 -0
- package/src/ontology-storage/materializations.ts +81 -7
- package/src/pg-ai-cost-storage.ts +119 -15
- package/src/pg-client.ts +13 -6
- package/src/storage-errors.ts +28 -1
package/dist/index.js
CHANGED
|
@@ -62,6 +62,7 @@ var LOCK_TABLES = [
|
|
|
62
62
|
{ setting: "max_locks_per_transaction", table: "lock table", note: "" }
|
|
63
63
|
];
|
|
64
64
|
var OUT_OF_MEMORY = "53200";
|
|
65
|
+
var IN_FAILED_SQL_TRANSACTION = "25P02";
|
|
65
66
|
function pgErrorCode(error) {
|
|
66
67
|
if (error instanceof Error) {
|
|
67
68
|
const code = error.code;
|
|
@@ -86,6 +87,10 @@ function isRetryableTransactionConflict(error) {
|
|
|
86
87
|
const code = pgErrorCode(error);
|
|
87
88
|
return code !== undefined && RETRYABLE_TRANSACTION_CONFLICT_CODES.has(code);
|
|
88
89
|
}
|
|
90
|
+
function isLockConflict(error) {
|
|
91
|
+
const code = pgErrorCode(error);
|
|
92
|
+
return code === "55P03" || code === "40P01";
|
|
93
|
+
}
|
|
89
94
|
function isConnectionLost(error) {
|
|
90
95
|
const code = pgErrorCode(error);
|
|
91
96
|
if (code !== undefined && CONNECTION_LOST_CODES.has(code)) {
|
|
@@ -98,6 +103,9 @@ function explainPgError(error) {
|
|
|
98
103
|
if (isStorageUnavailable(error)) {
|
|
99
104
|
return storageUnavailable(error);
|
|
100
105
|
}
|
|
106
|
+
if (pgErrorCode(error) === IN_FAILED_SQL_TRANSACTION) {
|
|
107
|
+
return failedStatementLost(error);
|
|
108
|
+
}
|
|
101
109
|
if (!(error instanceof Error) || pgErrorCode(error) !== OUT_OF_MEMORY) {
|
|
102
110
|
return error;
|
|
103
111
|
}
|
|
@@ -109,6 +117,9 @@ function explainPgError(error) {
|
|
|
109
117
|
const { setting, table, note } = lockTable;
|
|
110
118
|
return new Error(`[SixbPg] PostgreSQL aborted the transaction because its shared ${table} is full ` + `(SQLSTATE 53200 "${error.message}"). The table holds ${setting} \xD7 (max_connections + ` + `max_prepared_transactions) entries shared by all sessions.${note} Raise ${setting} ` + "(applied at server restart) or shorten the transactions that overlap.", { cause: error });
|
|
111
119
|
}
|
|
120
|
+
function failedStatementLost(error) {
|
|
121
|
+
return new Error("[SixbPg] A statement was sent after an earlier statement of the same transaction failed, so " + "PostgreSQL rejected it (SQLSTATE 25P02) and the earlier error was lost. Code that handles " + "a failure inside a transaction must not send more SQL on it: let the error propagate.", { cause: error });
|
|
122
|
+
}
|
|
112
123
|
function isStorageUnavailable(error) {
|
|
113
124
|
const code = pgErrorCode(error);
|
|
114
125
|
return isConnectionLost(error) || code !== undefined && CONNECT_FAILED_CODES.has(code);
|
|
@@ -247,6 +258,11 @@ function rowToThreadRecord(row) {
|
|
|
247
258
|
...row.sandbox_params == null ? {} : {
|
|
248
259
|
sandboxParams: snapshotAgentThreadSandboxParams(typeof row.sandbox_params === "string" ? JSON.parse(row.sandbox_params) : row.sandbox_params)
|
|
249
260
|
},
|
|
261
|
+
...row.runs_on == null ? {} : { runsOn: row.runs_on },
|
|
262
|
+
...row.computer_id == null || row.computer_name == null || row.computer_folder == null ? {} : {
|
|
263
|
+
computer: { id: row.computer_id, name: row.computer_name, folder: row.computer_folder }
|
|
264
|
+
},
|
|
265
|
+
...row.moved_at == null ? {} : { movedAt: new Date(row.moved_at) },
|
|
250
266
|
status: row.status,
|
|
251
267
|
activeRunId: row.active_run_id,
|
|
252
268
|
lastMessageAt: row.last_message_at ? new Date(row.last_message_at) : undefined,
|
|
@@ -270,6 +286,7 @@ function rowToRunRecord(row) {
|
|
|
270
286
|
token: row.execution_token,
|
|
271
287
|
queueLeaseExpiresAt: new Date(row.execution_queue_lease_expires_at)
|
|
272
288
|
} : undefined,
|
|
289
|
+
cancelRequestedAt: row.cancel_requested_at ? new Date(row.cancel_requested_at) : undefined,
|
|
273
290
|
createdAt: new Date(row.created_at),
|
|
274
291
|
startedAt: row.started_at ? new Date(row.started_at) : undefined,
|
|
275
292
|
completedAt: row.completed_at ? new Date(row.completed_at) : undefined
|
|
@@ -815,6 +832,73 @@ class PgAgentContextCheckpointStore {
|
|
|
815
832
|
}
|
|
816
833
|
}
|
|
817
834
|
|
|
835
|
+
// src/agents/computers.ts
|
|
836
|
+
import { assertAgentComputerSession } from "@sixb/core/internal/agent-run-storage-provider";
|
|
837
|
+
|
|
838
|
+
class PgAgentComputerSessionStore {
|
|
839
|
+
sql;
|
|
840
|
+
constructor(sql) {
|
|
841
|
+
this.sql = sql;
|
|
842
|
+
}
|
|
843
|
+
async hold(input) {
|
|
844
|
+
assertAgentComputerSession(input, "SixbPg");
|
|
845
|
+
await this.sql`
|
|
846
|
+
INSERT INTO agent_computer_sessions (
|
|
847
|
+
project_id, owner_principal_type, owner_principal_id, id,
|
|
848
|
+
computer_id, computer_name, folder, expires_at
|
|
849
|
+
) VALUES (
|
|
850
|
+
${input.projectId}, ${input.ownerPrincipal.type}, ${input.ownerPrincipal.id}, ${input.id},
|
|
851
|
+
${input.computer.id}, ${input.computer.name}, ${input.computer.folder}, ${input.expiresAt}
|
|
852
|
+
)
|
|
853
|
+
ON CONFLICT (project_id, owner_principal_type, owner_principal_id, id)
|
|
854
|
+
DO UPDATE SET
|
|
855
|
+
computer_id = excluded.computer_id,
|
|
856
|
+
computer_name = excluded.computer_name,
|
|
857
|
+
folder = excluded.folder,
|
|
858
|
+
expires_at = excluded.expires_at
|
|
859
|
+
`;
|
|
860
|
+
}
|
|
861
|
+
async release(input) {
|
|
862
|
+
const [row] = await this.sql`
|
|
863
|
+
DELETE FROM agent_computer_sessions
|
|
864
|
+
WHERE project_id = ${input.projectId}
|
|
865
|
+
AND owner_principal_type = ${input.ownerPrincipal.type}
|
|
866
|
+
AND owner_principal_id = ${input.ownerPrincipal.id}
|
|
867
|
+
AND id = ${input.id}
|
|
868
|
+
RETURNING *
|
|
869
|
+
`;
|
|
870
|
+
return row ? rowToSession(row) : null;
|
|
871
|
+
}
|
|
872
|
+
async listLive(input) {
|
|
873
|
+
const rows = await this.sql`
|
|
874
|
+
SELECT * FROM agent_computer_sessions
|
|
875
|
+
WHERE project_id = ${input.projectId}
|
|
876
|
+
AND owner_principal_type = ${input.ownerPrincipal.type}
|
|
877
|
+
AND owner_principal_id = ${input.ownerPrincipal.id}
|
|
878
|
+
AND expires_at > ${input.now}
|
|
879
|
+
ORDER BY id
|
|
880
|
+
`;
|
|
881
|
+
return rows.map(rowToSession);
|
|
882
|
+
}
|
|
883
|
+
async removeLapsed(input) {
|
|
884
|
+
const rows = await this.sql`
|
|
885
|
+
DELETE FROM agent_computer_sessions
|
|
886
|
+
WHERE project_id = ${input.projectId} AND expires_at <= ${input.now}
|
|
887
|
+
RETURNING *
|
|
888
|
+
`;
|
|
889
|
+
return rows.map(rowToSession);
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
function rowToSession(row) {
|
|
893
|
+
return {
|
|
894
|
+
projectId: row.project_id,
|
|
895
|
+
id: row.id,
|
|
896
|
+
ownerPrincipal: { type: row.owner_principal_type, id: row.owner_principal_id },
|
|
897
|
+
computer: { id: row.computer_id, name: row.computer_name, folder: row.folder },
|
|
898
|
+
expiresAt: new Date(row.expires_at)
|
|
899
|
+
};
|
|
900
|
+
}
|
|
901
|
+
|
|
818
902
|
// src/agents/memory.ts
|
|
819
903
|
import {
|
|
820
904
|
agentMemoryConflicts,
|
|
@@ -1031,6 +1115,17 @@ class PgAgentMessageStore {
|
|
|
1031
1115
|
}
|
|
1032
1116
|
async deleteByRunId(input) {
|
|
1033
1117
|
return runPgTransaction(this.sql, async (tx) => {
|
|
1118
|
+
if (input.fence) {
|
|
1119
|
+
const [owner] = await tx`
|
|
1120
|
+
SELECT id FROM agent_runs
|
|
1121
|
+
WHERE project_id = ${input.projectId} AND id = ${input.fence.runId}
|
|
1122
|
+
AND status = 'running' AND execution_token = ${input.fence.executionToken}
|
|
1123
|
+
FOR SHARE
|
|
1124
|
+
`;
|
|
1125
|
+
if (!owner) {
|
|
1126
|
+
throw new AgentStorageError3("execution_lost", `[SixbPg] Execution token is no longer current on agent run '${input.fence.runId}'.`);
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1034
1129
|
const [thread] = await tx`
|
|
1035
1130
|
SELECT id FROM agent_threads
|
|
1036
1131
|
WHERE project_id = ${input.projectId} AND id = ${input.threadId}
|
|
@@ -1115,7 +1210,9 @@ import {
|
|
|
1115
1210
|
assertConversationAgentRunSpec,
|
|
1116
1211
|
assertConversationThreadRow,
|
|
1117
1212
|
assertCreateSubagentRunInput,
|
|
1213
|
+
assertStartsOnComputer,
|
|
1118
1214
|
assertSubagentThreadTakesRun,
|
|
1215
|
+
assertTurnInThreadPlace,
|
|
1119
1216
|
newSubagentThreadRecord,
|
|
1120
1217
|
subagentRunMatchesCreateInput
|
|
1121
1218
|
} from "@sixb/core/internal/agent-run-storage-provider";
|
|
@@ -1203,7 +1300,7 @@ class PgAgentRunStore {
|
|
|
1203
1300
|
try {
|
|
1204
1301
|
return await runPgTransaction(this.sql, async (tx) => {
|
|
1205
1302
|
const [thread] = await tx`
|
|
1206
|
-
SELECT id, kind, active_run_id FROM agent_threads
|
|
1303
|
+
SELECT id, kind, active_run_id, runs_on FROM agent_threads
|
|
1207
1304
|
WHERE project_id = ${input.projectId} AND id = ${input.threadId}
|
|
1208
1305
|
FOR UPDATE
|
|
1209
1306
|
`;
|
|
@@ -1214,6 +1311,7 @@ class PgAgentRunStore {
|
|
|
1214
1311
|
if (thread.active_run_id !== null) {
|
|
1215
1312
|
throw new AgentStorageError4("active_run_exists", `[SixbPg] Agent thread '${input.threadId}' already has an active run '${thread.active_run_id}'.`);
|
|
1216
1313
|
}
|
|
1314
|
+
assertTurnInThreadPlace(thread.runs_on, input.spec, input.threadId, "SixbPg");
|
|
1217
1315
|
const [row] = await tx`
|
|
1218
1316
|
INSERT INTO agent_runs (
|
|
1219
1317
|
project_id,
|
|
@@ -1347,7 +1445,14 @@ class PgAgentRunStore {
|
|
|
1347
1445
|
async start(input) {
|
|
1348
1446
|
const startedAt = input.startedAt ?? new Date;
|
|
1349
1447
|
return runPgTransaction(this.sql, async (tx) => {
|
|
1350
|
-
await this.lockStatus(tx, input.projectId, input.id, "queued");
|
|
1448
|
+
const run = await this.lockStatus(tx, input.projectId, input.id, "queued");
|
|
1449
|
+
if (input.onComputer) {
|
|
1450
|
+
const [thread] = await tx`
|
|
1451
|
+
SELECT computer_id AS id, computer_folder AS folder FROM agent_threads
|
|
1452
|
+
WHERE project_id = ${input.projectId} AND id = ${run.thread_id}
|
|
1453
|
+
`;
|
|
1454
|
+
assertStartsOnComputer(rowToRunRecord(run), thread?.id ? { id: thread.id, folder: thread.folder } : undefined, input.onComputer, "SixbPg");
|
|
1455
|
+
}
|
|
1351
1456
|
const [row] = await tx`
|
|
1352
1457
|
UPDATE agent_runs
|
|
1353
1458
|
SET
|
|
@@ -1382,7 +1487,10 @@ class PgAgentRunStore {
|
|
|
1382
1487
|
}
|
|
1383
1488
|
async reclaim(input) {
|
|
1384
1489
|
return runPgTransaction(this.sql, async (tx) => {
|
|
1385
|
-
await this.lockRunning(tx, input.projectId, input.id);
|
|
1490
|
+
const run = await this.lockRunning(tx, input.projectId, input.id);
|
|
1491
|
+
if (input.ifLapsedBy && run.execution_queue_lease_expires_at !== null && new Date(run.execution_queue_lease_expires_at).getTime() > input.ifLapsedBy.getTime()) {
|
|
1492
|
+
throw new AgentStorageError4("invalid_state", `[SixbPg] Agent run '${input.id}' is still held by its runner, whose lease has not lapsed.`);
|
|
1493
|
+
}
|
|
1386
1494
|
const [row] = await tx`
|
|
1387
1495
|
UPDATE agent_runs
|
|
1388
1496
|
SET
|
|
@@ -1413,6 +1521,22 @@ class PgAgentRunStore {
|
|
|
1413
1521
|
return rowToRunRecord(row);
|
|
1414
1522
|
});
|
|
1415
1523
|
}
|
|
1524
|
+
async requestCancel(input) {
|
|
1525
|
+
const [updated] = await this.sql`
|
|
1526
|
+
UPDATE agent_runs
|
|
1527
|
+
SET cancel_requested_at = ${input.at ?? new Date}
|
|
1528
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
1529
|
+
AND status = 'running' AND cancel_requested_at IS NULL
|
|
1530
|
+
RETURNING *
|
|
1531
|
+
`;
|
|
1532
|
+
if (updated)
|
|
1533
|
+
return rowToRunRecord(updated);
|
|
1534
|
+
const current = await this.getById(input);
|
|
1535
|
+
if (!current) {
|
|
1536
|
+
throw new AgentStorageError4("run_not_found", `[SixbPg] Agent run '${input.id}' not found for project '${input.projectId}'.`);
|
|
1537
|
+
}
|
|
1538
|
+
return current;
|
|
1539
|
+
}
|
|
1416
1540
|
async finish(input) {
|
|
1417
1541
|
const completedAt = input.completedAt ?? new Date;
|
|
1418
1542
|
const errorValue = input.status === "succeeded" || input.error === undefined ? null : serializeSixbFailure(input.error, AGENT_RUN_FAILURE_CODES2);
|
|
@@ -1513,6 +1637,9 @@ class PgAgentRunStore {
|
|
|
1513
1637
|
whereClauses.push(`parent_run_id = $${index++}`);
|
|
1514
1638
|
params.push(input.parentRunId);
|
|
1515
1639
|
}
|
|
1640
|
+
if (input.excludeComputerTurns) {
|
|
1641
|
+
whereClauses.push("spec->>'runsOn' IS DISTINCT FROM 'computer'");
|
|
1642
|
+
}
|
|
1516
1643
|
index = appendRunListFilters(whereClauses, params, index, input, "COALESCE(started_at, created_at)");
|
|
1517
1644
|
const { rows, total, hasMore } = await queryRunList({
|
|
1518
1645
|
sql: this.sql,
|
|
@@ -1688,9 +1815,15 @@ function threadNotFound(input) {
|
|
|
1688
1815
|
|
|
1689
1816
|
// src/agents/threads.ts
|
|
1690
1817
|
import {
|
|
1818
|
+
assertAgentThreadComputer,
|
|
1819
|
+
assertAssignableComputer,
|
|
1820
|
+
assertComputerPlace,
|
|
1691
1821
|
assertConversationThreadRow as assertConversationThreadRow2,
|
|
1822
|
+
assertMovableToCloud,
|
|
1823
|
+
movesComputerPlace,
|
|
1692
1824
|
participantsToAdd,
|
|
1693
1825
|
planAgentThreadFork,
|
|
1826
|
+
queuedTurnInCloud,
|
|
1694
1827
|
remapArtifactIds,
|
|
1695
1828
|
snapshotAgentThreadSandboxParams as snapshotAgentThreadSandboxParams2,
|
|
1696
1829
|
transitionAgentThreadSandbox
|
|
@@ -1720,9 +1853,14 @@ class PgAgentThreadStore {
|
|
|
1720
1853
|
});
|
|
1721
1854
|
}
|
|
1722
1855
|
async create(input) {
|
|
1856
|
+
assertComputerPlace(input.runsOn, input.computer, "SixbPg");
|
|
1723
1857
|
const sandboxParams = input.sandboxParams === undefined ? null : JSON.stringify(snapshotAgentThreadSandboxParams2(input.sandboxParams));
|
|
1724
1858
|
const createdAt = input.createdAt ?? new Date;
|
|
1725
1859
|
const updatedAt = input.updatedAt ?? createdAt;
|
|
1860
|
+
const runsOn = input.runsOn === undefined ? { column: this.sql``, value: this.sql`` } : {
|
|
1861
|
+
column: this.sql`runs_on, computer_id, computer_name, computer_folder,`,
|
|
1862
|
+
value: this.sql`${input.runsOn}, ${input.computer?.id ?? null}, ${input.computer?.name ?? null}, ${input.computer?.folder ?? null},`
|
|
1863
|
+
};
|
|
1726
1864
|
try {
|
|
1727
1865
|
const [row] = await this.sql`
|
|
1728
1866
|
INSERT INTO agent_threads (
|
|
@@ -1732,6 +1870,7 @@ class PgAgentThreadStore {
|
|
|
1732
1870
|
owner_principal_id,
|
|
1733
1871
|
title,
|
|
1734
1872
|
sandbox_params,
|
|
1873
|
+
${runsOn.column}
|
|
1735
1874
|
status,
|
|
1736
1875
|
created_at,
|
|
1737
1876
|
updated_at
|
|
@@ -1742,6 +1881,7 @@ class PgAgentThreadStore {
|
|
|
1742
1881
|
${input.ownerPrincipal.id},
|
|
1743
1882
|
${input.title ?? null},
|
|
1744
1883
|
${sandboxParams}::text::jsonb,
|
|
1884
|
+
${runsOn.value}
|
|
1745
1885
|
${input.status ?? "active"},
|
|
1746
1886
|
${createdAt},
|
|
1747
1887
|
${updatedAt}
|
|
@@ -1760,7 +1900,7 @@ class PgAgentThreadStore {
|
|
|
1760
1900
|
try {
|
|
1761
1901
|
return await runPgTransaction(this.sql, async (tx) => {
|
|
1762
1902
|
const [source] = await tx`
|
|
1763
|
-
SELECT id, kind, active_run_id FROM agent_threads
|
|
1903
|
+
SELECT id, kind, active_run_id, runs_on, moved_at FROM agent_threads
|
|
1764
1904
|
WHERE project_id = ${input.projectId} AND id = ${input.sourceThreadId}
|
|
1765
1905
|
FOR SHARE
|
|
1766
1906
|
`;
|
|
@@ -1790,6 +1930,7 @@ class PgAgentThreadStore {
|
|
|
1790
1930
|
ORDER BY summarized_through_seq DESC, created_at DESC, id DESC
|
|
1791
1931
|
`;
|
|
1792
1932
|
const plan = planAgentThreadFork(input, {
|
|
1933
|
+
workspace: { runsOn: source.runs_on, movedAt: source.moved_at },
|
|
1793
1934
|
...activeRun?.trigger_message_id ? { triggerMessageId: activeRun.trigger_message_id } : {},
|
|
1794
1935
|
messages: messages.map((message) => ({
|
|
1795
1936
|
id: message.id,
|
|
@@ -1811,6 +1952,7 @@ class PgAgentThreadStore {
|
|
|
1811
1952
|
status,
|
|
1812
1953
|
last_message_at,
|
|
1813
1954
|
message_count,
|
|
1955
|
+
moved_at,
|
|
1814
1956
|
created_at,
|
|
1815
1957
|
updated_at
|
|
1816
1958
|
) VALUES (
|
|
@@ -1824,6 +1966,7 @@ class PgAgentThreadStore {
|
|
|
1824
1966
|
${thread.status},
|
|
1825
1967
|
${thread.lastMessageAt ?? null},
|
|
1826
1968
|
${thread.messageCount},
|
|
1969
|
+
${thread.movedAt ?? null},
|
|
1827
1970
|
${thread.createdAt},
|
|
1828
1971
|
${thread.updatedAt}
|
|
1829
1972
|
)
|
|
@@ -1944,6 +2087,10 @@ class PgAgentThreadStore {
|
|
|
1944
2087
|
)`);
|
|
1945
2088
|
params.push(input.participantPrincipal.type, input.participantPrincipal.id);
|
|
1946
2089
|
}
|
|
2090
|
+
if (input.activeOnComputer) {
|
|
2091
|
+
whereClauses.push(`computer_id = $${index++} AND computer_folder = $${index++} AND active_run_id IS NOT NULL`);
|
|
2092
|
+
params.push(input.activeOnComputer.id, input.activeOnComputer.folder);
|
|
2093
|
+
}
|
|
1947
2094
|
const dir = input.order === "asc" ? "ASC" : "DESC";
|
|
1948
2095
|
const { rows, total, hasMore } = await queryAgentList({
|
|
1949
2096
|
sql: this.sql,
|
|
@@ -2006,10 +2153,78 @@ class PgAgentThreadStore {
|
|
|
2006
2153
|
`;
|
|
2007
2154
|
return rows.length > 0;
|
|
2008
2155
|
}
|
|
2156
|
+
async assignComputer(input) {
|
|
2157
|
+
assertAgentThreadComputer(input.computer, "SixbPg");
|
|
2158
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
2159
|
+
const { thread, active } = await lockThreadAndActiveRun(tx, input);
|
|
2160
|
+
assertAssignableComputer(thread, active, input.computer, input.id, "SixbPg");
|
|
2161
|
+
const movedAt = movesComputerPlace(thread, input.computer) ? input.movedAt ?? new Date : null;
|
|
2162
|
+
const [assigned] = await tx`
|
|
2163
|
+
UPDATE agent_threads
|
|
2164
|
+
SET computer_id = ${input.computer.id},
|
|
2165
|
+
computer_name = ${input.computer.name},
|
|
2166
|
+
computer_folder = ${input.computer.folder},
|
|
2167
|
+
moved_at = COALESCE(${movedAt}::timestamptz, moved_at)
|
|
2168
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
2169
|
+
RETURNING *
|
|
2170
|
+
`;
|
|
2171
|
+
return rowToThreadRecord(assigned);
|
|
2172
|
+
});
|
|
2173
|
+
}
|
|
2174
|
+
async moveToCloud(input) {
|
|
2175
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
2176
|
+
const { thread, active } = await lockThreadAndActiveRun(tx, input);
|
|
2177
|
+
assertMovableToCloud(thread, active, input.id, "SixbPg");
|
|
2178
|
+
const queued = queuedTurnInCloud(active);
|
|
2179
|
+
if (queued) {
|
|
2180
|
+
await tx`
|
|
2181
|
+
UPDATE agent_runs SET spec = ${JSON.stringify(queued.spec)}::text::jsonb
|
|
2182
|
+
WHERE project_id = ${input.projectId} AND id = ${queued.id}
|
|
2183
|
+
`;
|
|
2184
|
+
}
|
|
2185
|
+
const [movedThread] = await tx`
|
|
2186
|
+
UPDATE agent_threads
|
|
2187
|
+
SET runs_on = NULL, computer_id = NULL, computer_name = NULL, computer_folder = NULL,
|
|
2188
|
+
moved_at = ${input.movedAt ?? new Date}
|
|
2189
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
2190
|
+
RETURNING *
|
|
2191
|
+
`;
|
|
2192
|
+
return {
|
|
2193
|
+
thread: rowToThreadRecord(movedThread),
|
|
2194
|
+
...queued ? { queued } : {}
|
|
2195
|
+
};
|
|
2196
|
+
});
|
|
2197
|
+
}
|
|
2009
2198
|
async listParticipants(input) {
|
|
2010
2199
|
return selectParticipants(this.sql, input.projectId, [...new Set(input.threadIds)]);
|
|
2011
2200
|
}
|
|
2012
2201
|
}
|
|
2202
|
+
async function lockThreadAndActiveRun(tx, input) {
|
|
2203
|
+
for (let attempt = 0;; attempt += 1) {
|
|
2204
|
+
const [peek] = await tx`
|
|
2205
|
+
SELECT active_run_id FROM agent_threads
|
|
2206
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
2207
|
+
`;
|
|
2208
|
+
const runId = peek?.active_run_id ?? null;
|
|
2209
|
+
const runs = runId ? await tx`
|
|
2210
|
+
SELECT * FROM agent_runs WHERE project_id = ${input.projectId} AND id = ${runId}
|
|
2211
|
+
FOR UPDATE
|
|
2212
|
+
` : [];
|
|
2213
|
+
const [row] = await tx`
|
|
2214
|
+
SELECT * FROM agent_threads WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
2215
|
+
FOR UPDATE
|
|
2216
|
+
`;
|
|
2217
|
+
if ((row?.active_run_id ?? null) === runId) {
|
|
2218
|
+
return {
|
|
2219
|
+
thread: row ? rowToThreadRecord(row) : null,
|
|
2220
|
+
active: runs[0] ? rowToRunRecord(runs[0]) : null
|
|
2221
|
+
};
|
|
2222
|
+
}
|
|
2223
|
+
if (attempt >= 3) {
|
|
2224
|
+
throw new AgentStorageError6("invalid_state", `[SixbPg] Agent thread '${input.id}' kept changing turns while it moved: try again.`);
|
|
2225
|
+
}
|
|
2226
|
+
}
|
|
2227
|
+
}
|
|
2013
2228
|
async function selectParticipants(sql, projectId, threadIds) {
|
|
2014
2229
|
if (threadIds.length === 0) {
|
|
2015
2230
|
return [];
|
|
@@ -2074,6 +2289,65 @@ async function remapCopiedParts(sql, projectId, messageIds, copies) {
|
|
|
2074
2289
|
}
|
|
2075
2290
|
}
|
|
2076
2291
|
|
|
2292
|
+
// src/agents/token-rates.ts
|
|
2293
|
+
import {
|
|
2294
|
+
AGENT_TOKEN_RATE_SAMPLES,
|
|
2295
|
+
assertAgentTokenRateRecord
|
|
2296
|
+
} from "@sixb/core/internal/agent-run-storage-provider";
|
|
2297
|
+
class PgAgentTokenRateStore {
|
|
2298
|
+
sql;
|
|
2299
|
+
constructor(sql) {
|
|
2300
|
+
this.sql = sql;
|
|
2301
|
+
}
|
|
2302
|
+
async record(input) {
|
|
2303
|
+
assertAgentTokenRateRecord(input, "SixbPg");
|
|
2304
|
+
await runPgTransaction(this.sql, async (tx) => {
|
|
2305
|
+
await tx`
|
|
2306
|
+
INSERT INTO agent_token_rates (
|
|
2307
|
+
project_id, run_id, provider_id, model_id, bytes, tokens, measured_at
|
|
2308
|
+
) VALUES (
|
|
2309
|
+
${input.projectId}, ${input.runId}, ${input.providerId}, ${input.modelId},
|
|
2310
|
+
${input.bytes}, ${input.tokens}, ${input.measuredAt}
|
|
2311
|
+
)
|
|
2312
|
+
ON CONFLICT (project_id, run_id) DO UPDATE SET
|
|
2313
|
+
provider_id = EXCLUDED.provider_id, model_id = EXCLUDED.model_id,
|
|
2314
|
+
bytes = EXCLUDED.bytes, tokens = EXCLUDED.tokens, measured_at = EXCLUDED.measured_at
|
|
2315
|
+
`;
|
|
2316
|
+
await tx`
|
|
2317
|
+
DELETE FROM agent_token_rates
|
|
2318
|
+
WHERE project_id = ${input.projectId} AND provider_id = ${input.providerId}
|
|
2319
|
+
AND model_id = ${input.modelId}
|
|
2320
|
+
AND run_id NOT IN (
|
|
2321
|
+
SELECT run_id FROM agent_token_rates
|
|
2322
|
+
WHERE project_id = ${input.projectId} AND provider_id = ${input.providerId}
|
|
2323
|
+
AND model_id = ${input.modelId}
|
|
2324
|
+
ORDER BY measured_at DESC, run_id DESC
|
|
2325
|
+
LIMIT ${AGENT_TOKEN_RATE_SAMPLES}
|
|
2326
|
+
)
|
|
2327
|
+
`;
|
|
2328
|
+
});
|
|
2329
|
+
}
|
|
2330
|
+
async listLatest(input) {
|
|
2331
|
+
const rows = await this.sql`
|
|
2332
|
+
SELECT project_id, run_id, provider_id, model_id, bytes, tokens, measured_at
|
|
2333
|
+
FROM agent_token_rates
|
|
2334
|
+
WHERE project_id = ${input.projectId} AND provider_id = ${input.providerId}
|
|
2335
|
+
AND model_id = ${input.modelId}
|
|
2336
|
+
ORDER BY measured_at DESC, run_id DESC
|
|
2337
|
+
LIMIT ${AGENT_TOKEN_RATE_SAMPLES}
|
|
2338
|
+
`;
|
|
2339
|
+
return rows.map((row) => ({
|
|
2340
|
+
projectId: row.project_id,
|
|
2341
|
+
providerId: row.provider_id,
|
|
2342
|
+
modelId: row.model_id,
|
|
2343
|
+
runId: row.run_id,
|
|
2344
|
+
bytes: row.bytes,
|
|
2345
|
+
tokens: row.tokens,
|
|
2346
|
+
measuredAt: row.measured_at
|
|
2347
|
+
}));
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
|
|
2077
2351
|
// src/agents/agent-storage.ts
|
|
2078
2352
|
class PgAgentStorage {
|
|
2079
2353
|
threads;
|
|
@@ -2083,6 +2357,8 @@ class PgAgentStorage {
|
|
|
2083
2357
|
memory;
|
|
2084
2358
|
approvals;
|
|
2085
2359
|
approvalModes;
|
|
2360
|
+
computers;
|
|
2361
|
+
tokenRates;
|
|
2086
2362
|
constructor(options) {
|
|
2087
2363
|
this.threads = new PgAgentThreadStore(options.sql);
|
|
2088
2364
|
this.runs = new PgAgentRunStore(options.sql, options.executions);
|
|
@@ -2091,6 +2367,8 @@ class PgAgentStorage {
|
|
|
2091
2367
|
this.memory = new PgAgentMemoryStore(options.sql);
|
|
2092
2368
|
this.approvals = new PgAgentApprovalStore(options.sql);
|
|
2093
2369
|
this.approvalModes = new PgAgentApprovalModeStore(options.sql);
|
|
2370
|
+
this.computers = new PgAgentComputerSessionStore(options.sql);
|
|
2371
|
+
this.tokenRates = new PgAgentTokenRateStore(options.sql);
|
|
2094
2372
|
}
|
|
2095
2373
|
}
|
|
2096
2374
|
// src/auth-storage/auth-storage.ts
|
|
@@ -9732,6 +10010,164 @@ CREATE INDEX idx_agent_runs_paused_on_question
|
|
|
9732
10010
|
WHERE kind = 'subagent' AND finish_reason = 'awaiting_answer';
|
|
9733
10011
|
`;
|
|
9734
10012
|
|
|
10013
|
+
// src/migrations/082-agent-run-cancel-requests.sql
|
|
10014
|
+
var _082_agent_run_cancel_requests_default = `-- A cancel is recorded on the run, not only broadcast: a runner that never hears the broker learns
|
|
10015
|
+
-- of it at its next lease renewal, and the run token stops working at once.
|
|
10016
|
+
ALTER TABLE agent_runs ADD COLUMN cancel_requested_at TIMESTAMPTZ;
|
|
10017
|
+
`;
|
|
10018
|
+
|
|
10019
|
+
// src/migrations/083-agent-thread-runs-on.sql
|
|
10020
|
+
var _083_agent_thread_runs_on_default = `-- Where a conversation's turns run: NULL in the cloud, 'computer' on its owner's computer.
|
|
10021
|
+
ALTER TABLE agent_threads ADD COLUMN runs_on TEXT CHECK (runs_on IS NULL OR runs_on = 'computer');
|
|
10022
|
+
-- Which of its owner's computers runs a computer's thread: NULL until one claims its first turn.
|
|
10023
|
+
ALTER TABLE agent_threads ADD COLUMN computer_id TEXT;
|
|
10024
|
+
`;
|
|
10025
|
+
|
|
10026
|
+
// src/migrations/084-agent-token-rates.sql
|
|
10027
|
+
var _084_agent_token_rates_default = `-- What each model's tokenizer measured on a project's agent runs, one row per run: the bytes of a
|
|
10028
|
+
-- request's text as the Agent's context estimator weighs them, against the text tokens the
|
|
10029
|
+
-- provider counted. A record deletes the model's rows past its newest 20, so the table stays small.
|
|
10030
|
+
CREATE TABLE agent_token_rates (
|
|
10031
|
+
project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
|
|
10032
|
+
run_id TEXT NOT NULL CHECK (length(trim(run_id)) > 0),
|
|
10033
|
+
provider_id TEXT NOT NULL CHECK (length(trim(provider_id)) > 0),
|
|
10034
|
+
model_id TEXT NOT NULL CHECK (length(trim(model_id)) > 0),
|
|
10035
|
+
bytes INTEGER NOT NULL CHECK (bytes > 0),
|
|
10036
|
+
tokens INTEGER NOT NULL CHECK (tokens > 0),
|
|
10037
|
+
measured_at TIMESTAMPTZ NOT NULL,
|
|
10038
|
+
PRIMARY KEY (project_id, run_id)
|
|
10039
|
+
);
|
|
10040
|
+
|
|
10041
|
+
CREATE INDEX idx_agent_token_rates_model
|
|
10042
|
+
ON agent_token_rates (project_id, provider_id, model_id, measured_at DESC, run_id DESC);
|
|
10043
|
+
`;
|
|
10044
|
+
|
|
10045
|
+
// src/migrations/085-agent-computer-sessions.sql
|
|
10046
|
+
var _085_agent_computer_sessions_default = `-- Where a computer's thread runs: the computer's name and the folder there, beside its id. A
|
|
10047
|
+
-- computer kept before folders were has none until its owner moves the thread.
|
|
10048
|
+
ALTER TABLE agent_threads ADD COLUMN computer_name TEXT;
|
|
10049
|
+
ALTER TABLE agent_threads ADD COLUMN computer_folder TEXT;
|
|
10050
|
+
UPDATE agent_threads SET computer_id = NULL WHERE computer_id IS NOT NULL;
|
|
10051
|
+
-- When its turns last moved to where they run now: the files earlier turns used are not there.
|
|
10052
|
+
ALTER TABLE agent_threads ADD COLUMN moved_at TIMESTAMPTZ;
|
|
10053
|
+
|
|
10054
|
+
-- Each open sixb on its owner's computers, a lease it renews: while one is live, the threads that
|
|
10055
|
+
-- run on its computer in its folder are reachable there.
|
|
10056
|
+
CREATE TABLE agent_computer_sessions (
|
|
10057
|
+
project_id TEXT NOT NULL,
|
|
10058
|
+
owner_principal_type TEXT NOT NULL CHECK (owner_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
10059
|
+
owner_principal_id TEXT NOT NULL CHECK (length(owner_principal_id) > 0),
|
|
10060
|
+
id TEXT NOT NULL CHECK (length(id) > 0),
|
|
10061
|
+
computer_id TEXT NOT NULL,
|
|
10062
|
+
computer_name TEXT NOT NULL,
|
|
10063
|
+
folder TEXT NOT NULL,
|
|
10064
|
+
expires_at TIMESTAMPTZ NOT NULL,
|
|
10065
|
+
PRIMARY KEY (project_id, owner_principal_type, owner_principal_id, id)
|
|
10066
|
+
);
|
|
10067
|
+
CREATE INDEX agent_computer_sessions_by_expiry ON agent_computer_sessions (project_id, expires_at);
|
|
10068
|
+
|
|
10069
|
+
-- Each open sixb asks every 20 seconds which of its owner's threads here have a turn active.
|
|
10070
|
+
CREATE INDEX agent_threads_active_on_computer
|
|
10071
|
+
ON agent_threads (project_id, owner_principal_type, owner_principal_id, computer_id, computer_folder)
|
|
10072
|
+
WHERE active_run_id IS NOT NULL;
|
|
10073
|
+
`;
|
|
10074
|
+
|
|
10075
|
+
// src/migrations/086-partition-objects.sql
|
|
10076
|
+
var _086_partition_objects_default = `-- Partition objects by object type, so a query plans against its own type's indexes alone instead
|
|
10077
|
+
-- of every index of every type. Query indexes and extended statistics go with the old table:
|
|
10078
|
+
-- preparation builds them again on the new partitions, right after this migration.
|
|
10079
|
+
DO $$
|
|
10080
|
+
DECLARE
|
|
10081
|
+
item record;
|
|
10082
|
+
columns_list text;
|
|
10083
|
+
restore_constraints text[] := '{}';
|
|
10084
|
+
restore_indexes text[] := '{}';
|
|
10085
|
+
statement text;
|
|
10086
|
+
BEGIN
|
|
10087
|
+
-- Copying the rows takes as long as it takes, whatever timeout this role's queries run under.
|
|
10088
|
+
-- Waiting for the lock does not: while this waits, every other query waits behind it, so it gives
|
|
10089
|
+
-- way to a long query, and the migration fails cleanly if that query outlasts every attempt.
|
|
10090
|
+
PERFORM set_config('statement_timeout', '0', true);
|
|
10091
|
+
PERFORM set_config('lock_timeout', '3s', true);
|
|
10092
|
+
FOR attempt IN 1..5 LOOP
|
|
10093
|
+
BEGIN
|
|
10094
|
+
LOCK TABLE objects IN ACCESS EXCLUSIVE MODE;
|
|
10095
|
+
EXIT;
|
|
10096
|
+
EXCEPTION WHEN lock_not_available THEN
|
|
10097
|
+
IF attempt = 5 THEN RAISE; END IF;
|
|
10098
|
+
PERFORM pg_sleep(attempt * 0.5);
|
|
10099
|
+
END;
|
|
10100
|
+
END LOOP;
|
|
10101
|
+
|
|
10102
|
+
-- objects is held now. Tables that reference it may have readers, so wait for those a little
|
|
10103
|
+
-- longer, and fail cleanly rather than hold every query behind objects for good.
|
|
10104
|
+
PERFORM set_config('lock_timeout', '30s', true);
|
|
10105
|
+
|
|
10106
|
+
FOR item IN
|
|
10107
|
+
SELECT conrelid::regclass::text AS table_name, conname, pg_get_constraintdef(oid) AS definition
|
|
10108
|
+
FROM pg_constraint
|
|
10109
|
+
WHERE confrelid = 'objects'::regclass AND contype = 'f' AND conparentid = 0
|
|
10110
|
+
LOOP
|
|
10111
|
+
EXECUTE format('ALTER TABLE %s DROP CONSTRAINT %I', item.table_name, item.conname);
|
|
10112
|
+
restore_constraints := restore_constraints || format(
|
|
10113
|
+
'ALTER TABLE %s ADD CONSTRAINT %I %s', item.table_name, item.conname, item.definition
|
|
10114
|
+
);
|
|
10115
|
+
END LOOP;
|
|
10116
|
+
|
|
10117
|
+
SELECT string_agg(quote_ident(attname), ', ' ORDER BY attnum) INTO columns_list
|
|
10118
|
+
FROM pg_attribute
|
|
10119
|
+
WHERE attrelid = 'objects'::regclass AND attnum > 0 AND NOT attisdropped AND attgenerated = '';
|
|
10120
|
+
|
|
10121
|
+
-- The table's general indexes stay on the parent under their names; prepared ones do not.
|
|
10122
|
+
FOR item IN
|
|
10123
|
+
SELECT c.relname AS name, pg_get_indexdef(c.oid) AS definition
|
|
10124
|
+
FROM pg_index i
|
|
10125
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
10126
|
+
WHERE i.indrelid = 'objects'::regclass
|
|
10127
|
+
AND NOT i.indisprimary
|
|
10128
|
+
AND c.relname NOT LIKE 'sixb\\_query\\_%'
|
|
10129
|
+
AND c.relname NOT LIKE 'sixb\\_text\\_count\\_%'
|
|
10130
|
+
LOOP
|
|
10131
|
+
restore_indexes := restore_indexes || item.definition;
|
|
10132
|
+
EXECUTE format('DROP INDEX %I', item.name);
|
|
10133
|
+
END LOOP;
|
|
10134
|
+
|
|
10135
|
+
ALTER TABLE objects RENAME TO objects_unpartitioned;
|
|
10136
|
+
ALTER INDEX objects_pkey RENAME TO objects_unpartitioned_pkey;
|
|
10137
|
+
|
|
10138
|
+
CREATE TABLE objects (
|
|
10139
|
+
LIKE objects_unpartitioned
|
|
10140
|
+
INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE INCLUDING COMMENTS,
|
|
10141
|
+
PRIMARY KEY (project_id, object_type_id, primary_id)
|
|
10142
|
+
) PARTITION BY LIST (object_type_id);
|
|
10143
|
+
|
|
10144
|
+
-- Holds a type until its first preparation gives it a partition.
|
|
10145
|
+
CREATE TABLE objects_default PARTITION OF objects DEFAULT;
|
|
10146
|
+
|
|
10147
|
+
FOR item IN SELECT DISTINCT object_type_id AS id FROM objects_unpartitioned LOOP
|
|
10148
|
+
EXECUTE format(
|
|
10149
|
+
'CREATE TABLE %I PARTITION OF objects FOR VALUES IN (%L)',
|
|
10150
|
+
'objects_' || left(md5(item.id), 24),
|
|
10151
|
+
item.id
|
|
10152
|
+
);
|
|
10153
|
+
END LOOP;
|
|
10154
|
+
|
|
10155
|
+
EXECUTE format(
|
|
10156
|
+
'INSERT INTO objects (%s) SELECT %s FROM objects_unpartitioned', columns_list, columns_list
|
|
10157
|
+
);
|
|
10158
|
+
DROP TABLE objects_unpartitioned;
|
|
10159
|
+
|
|
10160
|
+
FOREACH statement IN ARRAY restore_indexes LOOP
|
|
10161
|
+
EXECUTE statement;
|
|
10162
|
+
END LOOP;
|
|
10163
|
+
FOREACH statement IN ARRAY restore_constraints LOOP
|
|
10164
|
+
EXECUTE statement;
|
|
10165
|
+
END LOOP;
|
|
10166
|
+
END $$;
|
|
10167
|
+
|
|
10168
|
+
ANALYZE objects;
|
|
10169
|
+
`;
|
|
10170
|
+
|
|
9735
10171
|
// src/migrations.ts
|
|
9736
10172
|
var POSTGRES_STORAGE_ADAPTER_ID = "SixbPostgresStorage";
|
|
9737
10173
|
function pgSql(id, sqlText) {
|
|
@@ -9979,40 +10415,136 @@ var postgresStorageMigrations = defineMigrations({
|
|
|
9979
10415
|
pgSql("078-agent-approvals", _078_agent_approvals_default),
|
|
9980
10416
|
pgSql("079-agent-approval-resume", _079_agent_approval_resume_default),
|
|
9981
10417
|
pgSql("080-subagent-threads", _080_subagent_threads_default),
|
|
9982
|
-
pgSql("081-subagent-questions", _081_subagent_questions_default)
|
|
10418
|
+
pgSql("081-subagent-questions", _081_subagent_questions_default),
|
|
10419
|
+
pgSql("082-agent-run-cancel-requests", _082_agent_run_cancel_requests_default),
|
|
10420
|
+
pgSql("083-agent-thread-runs-on", _083_agent_thread_runs_on_default),
|
|
10421
|
+
pgSql("084-agent-token-rates", _084_agent_token_rates_default),
|
|
10422
|
+
pgSql("085-agent-computer-sessions", _085_agent_computer_sessions_default),
|
|
10423
|
+
pgSql("086-partition-objects", _086_partition_objects_default)
|
|
9983
10424
|
]
|
|
9984
10425
|
});
|
|
9985
10426
|
|
|
9986
|
-
// src/objects/
|
|
9987
|
-
import {
|
|
10427
|
+
// src/objects/query-indexes.ts
|
|
10428
|
+
import { createHash as createHash4 } from "crypto";
|
|
10429
|
+
import { linkEndpointObjectTypeIds as linkEndpointObjectTypeIds2 } from "@sixb/core/internal/query";
|
|
9988
10430
|
|
|
9989
|
-
// src/objects/
|
|
9990
|
-
|
|
9991
|
-
|
|
9992
|
-
|
|
9993
|
-
|
|
9994
|
-
|
|
9995
|
-
|
|
9996
|
-
|
|
9997
|
-
|
|
9998
|
-
|
|
9999
|
-
|
|
10000
|
-
|
|
10001
|
-
|
|
10431
|
+
// src/objects/partitions.ts
|
|
10432
|
+
import { createHash as createHash3 } from "crypto";
|
|
10433
|
+
|
|
10434
|
+
// src/objects/sql-text.ts
|
|
10435
|
+
function identifier(value) {
|
|
10436
|
+
return `"${value.replace(/"/g, '""')}"`;
|
|
10437
|
+
}
|
|
10438
|
+
function literal(value) {
|
|
10439
|
+
if (value.includes("\x00"))
|
|
10440
|
+
throw new Error("[SixbPg] SQL literals cannot contain NUL");
|
|
10441
|
+
return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`;
|
|
10442
|
+
}
|
|
10443
|
+
|
|
10444
|
+
// src/objects/partitions.ts
|
|
10445
|
+
var LOCK_TIMEOUT = "1s";
|
|
10446
|
+
var LOCK_ATTEMPTS = 120;
|
|
10447
|
+
var LOCK_RETRY_MS = 1000;
|
|
10448
|
+
function objectPartitionName(objectTypeId) {
|
|
10449
|
+
return `objects_${createHash3("md5").update(objectTypeId).digest("hex").slice(0, 24)}`;
|
|
10450
|
+
}
|
|
10451
|
+
async function ensureObjectPartition(connection, schema, objectTypeId) {
|
|
10452
|
+
const name = objectPartitionName(objectTypeId);
|
|
10453
|
+
const partition = `${identifier(schema)}.${identifier(name)}`;
|
|
10454
|
+
const [existing] = await connection`
|
|
10455
|
+
SELECT to_regclass(${partition})::text AS found`;
|
|
10456
|
+
if (existing?.found)
|
|
10457
|
+
return name;
|
|
10458
|
+
for (let attempt = 1;; attempt++) {
|
|
10002
10459
|
try {
|
|
10003
|
-
|
|
10004
|
-
|
|
10005
|
-
|
|
10006
|
-
if (
|
|
10007
|
-
|
|
10008
|
-
|
|
10009
|
-
|
|
10460
|
+
await createObjectPartition(connection, schema, partition, objectTypeId);
|
|
10461
|
+
return name;
|
|
10462
|
+
} catch (error) {
|
|
10463
|
+
if (!isLockConflict(error))
|
|
10464
|
+
throw error;
|
|
10465
|
+
if (attempt === LOCK_ATTEMPTS) {
|
|
10466
|
+
throw new Error(`[SixbPg] Could not add a partition for object type '${objectTypeId}': other queries held their locks through ${LOCK_ATTEMPTS} attempts. Retry when they finish.`, { cause: error });
|
|
10467
|
+
}
|
|
10468
|
+
}
|
|
10469
|
+
await Bun.sleep(LOCK_RETRY_MS);
|
|
10470
|
+
}
|
|
10471
|
+
}
|
|
10472
|
+
async function createObjectPartition(connection, schema, partition, objectTypeId) {
|
|
10473
|
+
const parent = `${identifier(schema)}.objects`;
|
|
10474
|
+
const fallback = `${identifier(schema)}.objects_default`;
|
|
10475
|
+
const type = literal(objectTypeId);
|
|
10476
|
+
const hasRows = async () => {
|
|
10477
|
+
const [row] = await connection.unsafe(`SELECT 1 FROM ${fallback} WHERE object_type_id = ${type} LIMIT 1`);
|
|
10478
|
+
return row !== undefined;
|
|
10479
|
+
};
|
|
10480
|
+
const guarded = (run) => runPgTransactionOn(connection, async () => {
|
|
10481
|
+
await connection.unsafe(`SET LOCAL lock_timeout = '${LOCK_TIMEOUT}'`);
|
|
10482
|
+
await connection.unsafe("SET LOCAL statement_timeout = 0");
|
|
10483
|
+
await run();
|
|
10484
|
+
});
|
|
10485
|
+
let moving = false;
|
|
10486
|
+
await guarded(async () => {
|
|
10487
|
+
moving = await hasRows();
|
|
10488
|
+
});
|
|
10489
|
+
await guarded(async () => {
|
|
10490
|
+
const references = await referencesToObjects(connection, parent);
|
|
10491
|
+
await connection.unsafe(`LOCK TABLE ONLY ${parent} IN ${moving ? "ACCESS EXCLUSIVE" : "SHARE UPDATE EXCLUSIVE"} MODE`);
|
|
10492
|
+
for (const { table_name: table } of references) {
|
|
10493
|
+
await connection.unsafe(`LOCK TABLE ${table} IN SHARE ROW EXCLUSIVE MODE`);
|
|
10494
|
+
}
|
|
10495
|
+
await connection.unsafe(`LOCK TABLE ${fallback} IN ACCESS EXCLUSIVE MODE`);
|
|
10496
|
+
if (!moving && await hasRows()) {
|
|
10497
|
+
throw Object.assign(new Error("[SixbPg] A row arrived while adding its partition."), {
|
|
10498
|
+
code: "55P03"
|
|
10499
|
+
});
|
|
10010
10500
|
}
|
|
10501
|
+
await connection.unsafe(`CREATE TABLE ${partition} (LIKE ${parent} INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE)`);
|
|
10502
|
+
if (moving)
|
|
10503
|
+
await moveRowsFromDefault(connection, parent, fallback, partition, type, references);
|
|
10504
|
+
await connection.unsafe(`ALTER TABLE ${parent} ATTACH PARTITION ${partition} FOR VALUES IN (${type})`);
|
|
10505
|
+
if (moving) {
|
|
10506
|
+
for (const reference of references) {
|
|
10507
|
+
await connection.unsafe(`ALTER TABLE ${reference.table_name} ADD CONSTRAINT ${identifier(reference.name)} ${reference.definition}`);
|
|
10508
|
+
}
|
|
10509
|
+
}
|
|
10510
|
+
});
|
|
10511
|
+
if (moving)
|
|
10512
|
+
await guarded(() => connection.unsafe(`ANALYZE ${partition}`).then(() => {
|
|
10513
|
+
return;
|
|
10514
|
+
}));
|
|
10515
|
+
}
|
|
10516
|
+
function referencesToObjects(connection, parent) {
|
|
10517
|
+
return connection`
|
|
10518
|
+
SELECT conrelid::regclass::text AS table_name, conname AS name,
|
|
10519
|
+
pg_get_constraintdef(oid) AS definition
|
|
10520
|
+
FROM pg_constraint
|
|
10521
|
+
WHERE confrelid = ${parent}::regclass AND contype = 'f' AND conparentid = 0`;
|
|
10522
|
+
}
|
|
10523
|
+
async function moveRowsFromDefault(connection, parent, fallback, partition, type, references) {
|
|
10524
|
+
for (const reference of references) {
|
|
10525
|
+
await connection.unsafe(`ALTER TABLE ${reference.table_name} DROP CONSTRAINT ${identifier(reference.name)}`);
|
|
10011
10526
|
}
|
|
10527
|
+
const columns = await connection`
|
|
10528
|
+
SELECT attname AS name FROM pg_attribute
|
|
10529
|
+
WHERE attrelid = ${parent}::regclass AND attnum > 0 AND NOT attisdropped AND attgenerated = ''
|
|
10530
|
+
ORDER BY attnum`;
|
|
10531
|
+
const list = columns.map((column) => identifier(column.name)).join(", ");
|
|
10532
|
+
await connection.unsafe(`INSERT INTO ${partition} (${list}) SELECT ${list} FROM ${fallback} WHERE object_type_id = ${type}`);
|
|
10533
|
+
await connection.unsafe(`DELETE FROM ${fallback} WHERE object_type_id = ${type}`);
|
|
10012
10534
|
}
|
|
10013
10535
|
|
|
10014
|
-
// src/objects/
|
|
10015
|
-
|
|
10536
|
+
// src/objects/preparation-lock.ts
|
|
10537
|
+
async function lockPgPreparation(sql, key) {
|
|
10538
|
+
let delay = 25;
|
|
10539
|
+
while (true) {
|
|
10540
|
+
const [row] = await sql`
|
|
10541
|
+
SELECT pg_try_advisory_lock(hashtextextended(${key}, 0)) AS acquired`;
|
|
10542
|
+
if (row?.acquired)
|
|
10543
|
+
return;
|
|
10544
|
+
await Bun.sleep(delay);
|
|
10545
|
+
delay = Math.min(delay * 2, 500);
|
|
10546
|
+
}
|
|
10547
|
+
}
|
|
10016
10548
|
|
|
10017
10549
|
// src/objects/query-compiler.ts
|
|
10018
10550
|
import {
|
|
@@ -10021,6 +10553,7 @@ import {
|
|
|
10021
10553
|
import {
|
|
10022
10554
|
hasVectorProfile,
|
|
10023
10555
|
isVectorProfileQuery,
|
|
10556
|
+
linkEndpointObjectTypeIds,
|
|
10024
10557
|
normalizeVector,
|
|
10025
10558
|
vectorSearchCandidateLimit
|
|
10026
10559
|
} from "@sixb/core/internal/query";
|
|
@@ -10038,10 +10571,13 @@ function compilePgObjectQuery(projectId, query, options = {}) {
|
|
|
10038
10571
|
}));
|
|
10039
10572
|
if (options.sortIndexBound && source === DEFAULT_OBJECT_QUERY_SOURCE) {
|
|
10040
10573
|
let bounded = query;
|
|
10041
|
-
|
|
10574
|
+
const expansions = [];
|
|
10575
|
+
while (bounded.kind === "expand") {
|
|
10576
|
+
expansions.unshift(bounded.expansions);
|
|
10042
10577
|
bounded = bounded.input;
|
|
10578
|
+
}
|
|
10043
10579
|
if (bounded.kind === "page" || bounded.kind === "limit") {
|
|
10044
|
-
const branches = [options.sortIndexBound, `NOT (${options.sortIndexBound})`].map((startPredicate, branch) => compileObjectQueryInternal(projectId,
|
|
10580
|
+
const branches = [options.sortIndexBound, `NOT (${options.sortIndexBound})`].map((startPredicate, branch) => compileObjectQueryInternal(projectId, bounded, {
|
|
10045
10581
|
probeLimit: options.includeTotal === false,
|
|
10046
10582
|
source,
|
|
10047
10583
|
startPredicate,
|
|
@@ -10049,11 +10585,17 @@ function compilePgObjectQuery(projectId, query, options = {}) {
|
|
|
10049
10585
|
correlatedTraversal: options.correlatedTraversal
|
|
10050
10586
|
}));
|
|
10051
10587
|
const limit = bounded.kind === "page" ? bounded.pageSize + 1 : bounded.limit + (options.includeTotal === false ? 1 : 0);
|
|
10052
|
-
|
|
10053
|
-
...compiled,
|
|
10588
|
+
let rows = {
|
|
10054
10589
|
sql: `SELECT * FROM ((${branches[0].sql}) UNION ALL (${branches[1].sql})) AS input ORDER BY ${compiled.order.sql} LIMIT ?`,
|
|
10055
10590
|
args: [...branches[0].args, ...branches[1].args, ...compiled.order.args, limit]
|
|
10056
10591
|
};
|
|
10592
|
+
for (const expansion of expansions) {
|
|
10593
|
+
rows = expandRows({ ...rows, order: compiled.order }, expansion, {
|
|
10594
|
+
probeLimit: options.includeTotal === false,
|
|
10595
|
+
source
|
|
10596
|
+
});
|
|
10597
|
+
}
|
|
10598
|
+
compiled = { ...compiled, ...rows };
|
|
10057
10599
|
}
|
|
10058
10600
|
}
|
|
10059
10601
|
const traversal = !options.correlatedTraversal && boundedTraversal(query);
|
|
@@ -10162,7 +10704,7 @@ function compileObjectQueryInternal(projectId, query, ctx) {
|
|
|
10162
10704
|
case "page":
|
|
10163
10705
|
return compilePage(projectId, query.input, query.pageSize, query.pageToken, ctx);
|
|
10164
10706
|
case "traverse":
|
|
10165
|
-
return compileTraversal(projectId, query
|
|
10707
|
+
return compileTraversal(projectId, query, ctx);
|
|
10166
10708
|
case "set":
|
|
10167
10709
|
return compileSet(projectId, query.op, query.inputs, ctx);
|
|
10168
10710
|
case "project":
|
|
@@ -10230,7 +10772,7 @@ function compileStart(projectId, query, ctx) {
|
|
|
10230
10772
|
const order = compileOrder(identityOrderFields());
|
|
10231
10773
|
const objects = pgObjectsOfType(ctx.source, query.objectTypeId);
|
|
10232
10774
|
const sql = `
|
|
10233
|
-
SELECT
|
|
10775
|
+
SELECT *
|
|
10234
10776
|
FROM ${objects.table}
|
|
10235
10777
|
WHERE project_id = ? AND object_type_id = ? ${objects.filter}
|
|
10236
10778
|
${ctx.startPredicate ? `AND (${ctx.startPredicate})` : ""}
|
|
@@ -10268,7 +10810,7 @@ function compileRefs(projectId, query, ctx) {
|
|
|
10268
10810
|
FROM jsonb_array_elements(?::text::jsonb) AS ref(value)
|
|
10269
10811
|
`;
|
|
10270
10812
|
const sql = `
|
|
10271
|
-
SELECT selected
|
|
10813
|
+
SELECT selected.*
|
|
10272
10814
|
FROM (${requested}) AS requested
|
|
10273
10815
|
JOIN ${ctx.source.objectsTable} AS selected
|
|
10274
10816
|
ON selected.project_id = ?
|
|
@@ -10347,7 +10889,6 @@ function compileSort(projectId, inputQuery, fields, ctx) {
|
|
|
10347
10889
|
input.object_type_id,
|
|
10348
10890
|
input.primary_id,
|
|
10349
10891
|
input.properties,
|
|
10350
|
-
input.properties AS _cursor_properties,
|
|
10351
10892
|
input.created_at,
|
|
10352
10893
|
input.updated_at,
|
|
10353
10894
|
input.version,
|
|
@@ -10441,17 +10982,17 @@ function compilePage(projectId, inputQuery, rawPageSize, pageToken, ctx) {
|
|
|
10441
10982
|
}
|
|
10442
10983
|
};
|
|
10443
10984
|
}
|
|
10444
|
-
function compileTraversal(projectId,
|
|
10445
|
-
const input = compileObjectQueryInternal(projectId,
|
|
10985
|
+
function compileTraversal(projectId, traversal, ctx) {
|
|
10986
|
+
const input = compileObjectQueryInternal(projectId, traversal.input, {
|
|
10446
10987
|
...exactContext(ctx),
|
|
10447
10988
|
unordered: true,
|
|
10448
10989
|
startPredicate: undefined,
|
|
10449
10990
|
isolateSortOverflow: false
|
|
10450
10991
|
});
|
|
10451
|
-
const relation = compileTraversalRelation(projectId, input,
|
|
10992
|
+
const relation = compileTraversalRelation(projectId, input, traversal, ctx.source, ctx.correlatedTraversal, ctx.startPredicate);
|
|
10452
10993
|
const order = compileOrder(identityOrderFields());
|
|
10453
10994
|
return {
|
|
10454
|
-
sql: `SELECT selected
|
|
10995
|
+
sql: `SELECT selected.* ${relation.sql} ${ctx.unordered ? "" : `ORDER BY ${order.sql}`} ${ctx.isolateSortOverflow ? "OFFSET 0" : ""}`,
|
|
10455
10996
|
args: [...relation.args, ...ctx.unordered ? [] : order.args],
|
|
10456
10997
|
totalSql: `SELECT COUNT(*)::bigint AS total ${relation.sql}`,
|
|
10457
10998
|
totalArgs: relation.args,
|
|
@@ -10463,15 +11004,15 @@ function compileTraversal(projectId, inputQuery, linkId, direction, sourceObject
|
|
|
10463
11004
|
}
|
|
10464
11005
|
};
|
|
10465
11006
|
}
|
|
10466
|
-
function compileTraversalRelation(projectId, input,
|
|
10467
|
-
const incoming = direction === "incoming";
|
|
11007
|
+
function compileTraversalRelation(projectId, input, traversal, source, correlated = false, endpointPredicate) {
|
|
11008
|
+
const incoming = traversal.direction === "incoming";
|
|
10468
11009
|
const parent = incoming ? "target" : "source";
|
|
10469
11010
|
const endpoint = incoming ? "source" : "target";
|
|
10470
|
-
const
|
|
11011
|
+
const endpointType = compileEndpointTypes("selected", linkEndpointObjectTypeIds(traversal));
|
|
10471
11012
|
return {
|
|
10472
11013
|
sql: `FROM ${source.objectsTable} AS selected
|
|
10473
11014
|
WHERE selected.project_id = ?
|
|
10474
|
-
${
|
|
11015
|
+
${endpointType.sql}
|
|
10475
11016
|
${endpointPredicate ? `AND (${endpointPredicate})` : ""}
|
|
10476
11017
|
AND EXISTS (
|
|
10477
11018
|
SELECT 1 FROM (${input.sql}) AS input
|
|
@@ -10485,12 +11026,15 @@ function compileTraversalRelation(projectId, input, linkId, direction, sourceObj
|
|
|
10485
11026
|
AND edge.link_id = ?
|
|
10486
11027
|
${correlated ? "OFFSET 0" : ""}
|
|
10487
11028
|
)`,
|
|
10488
|
-
args: [
|
|
10489
|
-
|
|
10490
|
-
|
|
10491
|
-
|
|
10492
|
-
|
|
10493
|
-
]
|
|
11029
|
+
args: [projectId, ...endpointType.args, ...input.args, traversal.linkId]
|
|
11030
|
+
};
|
|
11031
|
+
}
|
|
11032
|
+
function compileEndpointTypes(alias, objectTypeIds) {
|
|
11033
|
+
if (!objectTypeIds?.length)
|
|
11034
|
+
return { sql: "", args: [] };
|
|
11035
|
+
return objectTypeIds.length === 1 ? { sql: `AND ${alias}.object_type_id = ?`, args: [objectTypeIds[0]] } : {
|
|
11036
|
+
sql: `AND ${alias}.object_type_id IN (${objectTypeIds.map(() => "?").join(", ")})`,
|
|
11037
|
+
args: [...objectTypeIds]
|
|
10494
11038
|
};
|
|
10495
11039
|
}
|
|
10496
11040
|
function boundedTraversal(query, bounded = false) {
|
|
@@ -10528,16 +11072,8 @@ function compileTraversalProbe(projectId, traversal, source) {
|
|
|
10528
11072
|
}
|
|
10529
11073
|
function compileExpand(projectId, inputQuery, expansions, ctx) {
|
|
10530
11074
|
const input = compileObjectQueryInternal(projectId, inputQuery, exactContext(ctx));
|
|
10531
|
-
const expand = compileExpansionsObject(expansions, { project: "input.project_id", type: "input.object_type_id", id: "input.primary_id" }, "", ctx);
|
|
10532
|
-
const inputOrder = compileOrder(input.order.fields, "input", "input._cursor_properties");
|
|
10533
|
-
const sql = `
|
|
10534
|
-
SELECT input.*, ${expand.sql} AS _expand
|
|
10535
|
-
FROM (${input.sql}) AS input
|
|
10536
|
-
ORDER BY ${inputOrder.sql}
|
|
10537
|
-
`;
|
|
10538
11075
|
return {
|
|
10539
|
-
|
|
10540
|
-
args: [...expand.args, ...input.args, ...inputOrder.args],
|
|
11076
|
+
...expandRows(input, expansions, ctx),
|
|
10541
11077
|
totalSql: input.totalSql,
|
|
10542
11078
|
totalArgs: input.totalArgs,
|
|
10543
11079
|
vectorProbe: input.vectorProbe,
|
|
@@ -10548,6 +11084,18 @@ function compileExpand(projectId, inputQuery, expansions, ctx) {
|
|
|
10548
11084
|
nextPageToken: input.nextPageToken
|
|
10549
11085
|
};
|
|
10550
11086
|
}
|
|
11087
|
+
function expandRows(input, expansions, ctx) {
|
|
11088
|
+
const expand = compileExpansionsObject(expansions, { project: "input.project_id", type: "input.object_type_id", id: "input.primary_id" }, "", ctx);
|
|
11089
|
+
const inputOrder = compileOrder(input.order.fields, "input", `input.${input.order.propertyColumn}`);
|
|
11090
|
+
return {
|
|
11091
|
+
sql: `
|
|
11092
|
+
SELECT input.*, ${expand.sql} AS _expand
|
|
11093
|
+
FROM (${input.sql}) AS input
|
|
11094
|
+
ORDER BY ${inputOrder.sql}
|
|
11095
|
+
`,
|
|
11096
|
+
args: [...expand.args, ...input.args, ...inputOrder.args]
|
|
11097
|
+
};
|
|
11098
|
+
}
|
|
10551
11099
|
function compileExpansionsObject(expansions, parent, pathPrefix, ctx) {
|
|
10552
11100
|
const parts = [];
|
|
10553
11101
|
const args = [];
|
|
@@ -10576,31 +11124,36 @@ function compileExpansionValue(expansion, parent, path, ctx) {
|
|
|
10576
11124
|
`${parentId} = ${parent.id}`,
|
|
10577
11125
|
`${edge}.link_id = ?::text`
|
|
10578
11126
|
];
|
|
10579
|
-
const
|
|
11127
|
+
const endpointType = compileEndpointTypes(target, expansion.endpointObjectTypeIds);
|
|
11128
|
+
const whereArgs = [...endpointType.args, expansion.linkId];
|
|
10580
11129
|
if (incoming && expansion.sourceObjectTypeId !== undefined) {
|
|
10581
11130
|
whereParts.push(`${edge}.source_type_id = ?::text`);
|
|
10582
11131
|
whereArgs.push(expansion.sourceObjectTypeId);
|
|
10583
11132
|
}
|
|
10584
|
-
const
|
|
10585
|
-
SELECT ${child.sql} AS elem, row_number() OVER (ORDER BY ${order.sql}) AS _ord
|
|
11133
|
+
const from = `
|
|
10586
11134
|
FROM ${ctx.source.linksTable} AS ${edge}
|
|
10587
11135
|
JOIN ${ctx.source.objectsTable} AS ${target}
|
|
10588
11136
|
ON ${target}.project_id = ${edge}.project_id
|
|
10589
11137
|
AND ${target}.object_type_id = ${neighborType}
|
|
10590
11138
|
AND ${target}.primary_id = ${neighborId}
|
|
11139
|
+
${endpointType.sql}
|
|
10591
11140
|
WHERE ${whereParts.join(" AND ")}
|
|
10592
11141
|
`;
|
|
10593
|
-
const innerArgs = [...child.args, ...order.args, ...whereArgs];
|
|
10594
11142
|
if (expansion.cardinality === "one") {
|
|
10595
11143
|
return {
|
|
10596
|
-
sql: `(SELECT ${
|
|
10597
|
-
args:
|
|
11144
|
+
sql: `(SELECT ${child.sql} ${from} ORDER BY ${order.sql} LIMIT 1)`,
|
|
11145
|
+
args: [...child.args, ...whereArgs, ...order.args]
|
|
10598
11146
|
};
|
|
10599
11147
|
}
|
|
10600
11148
|
const limit = expansion.limit ?? DEFAULT_EXPANSION_FANOUT;
|
|
10601
11149
|
return {
|
|
10602
|
-
sql: `COALESCE((SELECT jsonb_agg(${ranked}.elem ORDER BY ${ranked}._ord) FROM (
|
|
10603
|
-
|
|
11150
|
+
sql: `COALESCE((SELECT jsonb_agg(${ranked}.elem ORDER BY ${ranked}._ord) FROM (
|
|
11151
|
+
SELECT ${child.sql} AS elem, row_number() OVER (ORDER BY ${order.sql}) AS _ord
|
|
11152
|
+
${from}
|
|
11153
|
+
ORDER BY ${order.sql}
|
|
11154
|
+
LIMIT ?
|
|
11155
|
+
) AS ${ranked}), '[]'::jsonb)`,
|
|
11156
|
+
args: [...child.args, ...order.args, ...whereArgs, ...order.args, limit]
|
|
10604
11157
|
};
|
|
10605
11158
|
}
|
|
10606
11159
|
function compileExpansionChildJson(expansion, edge, target, path, ctx) {
|
|
@@ -10646,7 +11199,7 @@ function compileSet(projectId, op, inputs, ctx) {
|
|
|
10646
11199
|
const order = compileOrder(identityOrderFields());
|
|
10647
11200
|
return {
|
|
10648
11201
|
sql: `
|
|
10649
|
-
SELECT
|
|
11202
|
+
SELECT *
|
|
10650
11203
|
FROM ${ctx.source.objectsTable}
|
|
10651
11204
|
WHERE 1 = 0
|
|
10652
11205
|
ORDER BY ${order.sql}
|
|
@@ -10667,7 +11220,7 @@ function compileSet(projectId, op, inputs, ctx) {
|
|
|
10667
11220
|
const order = compileOrder(identityOrderFields());
|
|
10668
11221
|
const selectedOrder = compileOrder(identityOrderFields(), "selected");
|
|
10669
11222
|
const sql = `
|
|
10670
|
-
SELECT selected
|
|
11223
|
+
SELECT selected.*
|
|
10671
11224
|
FROM (${identities.sql}) AS ids
|
|
10672
11225
|
JOIN ${ctx.source.objectsTable} AS selected
|
|
10673
11226
|
ON selected.project_id = ?
|
|
@@ -10693,7 +11246,8 @@ function compileProject(projectId, inputQuery, properties, ctx) {
|
|
|
10693
11246
|
if (!properties)
|
|
10694
11247
|
return input;
|
|
10695
11248
|
const projection = compileProjectionExpression(properties);
|
|
10696
|
-
const
|
|
11249
|
+
const original = `input.${input.order.propertyColumn}`;
|
|
11250
|
+
const inputOrder = compileOrder(input.order.fields, "input", original);
|
|
10697
11251
|
const outputOrder = compileOrder(input.order.fields, undefined, "_cursor_properties");
|
|
10698
11252
|
return {
|
|
10699
11253
|
sql: `
|
|
@@ -10702,7 +11256,7 @@ function compileProject(projectId, inputQuery, properties, ctx) {
|
|
|
10702
11256
|
input.object_type_id,
|
|
10703
11257
|
input.primary_id,
|
|
10704
11258
|
${projection.sql} AS properties,
|
|
10705
|
-
|
|
11259
|
+
${original} AS _cursor_properties,
|
|
10706
11260
|
input.created_at,
|
|
10707
11261
|
input.updated_at,
|
|
10708
11262
|
input.version,
|
|
@@ -10735,7 +11289,7 @@ function compileAggregateSource(projectId, query, source) {
|
|
|
10735
11289
|
return compileAggregateWhere(projectId, query.input, compileTextSearchPredicate(query.query, query.fields, query.fieldsByObjectType), source);
|
|
10736
11290
|
}
|
|
10737
11291
|
case "traverse":
|
|
10738
|
-
return compileAggregateTraversal(projectId, query
|
|
11292
|
+
return compileAggregateTraversal(projectId, query, source);
|
|
10739
11293
|
case "set":
|
|
10740
11294
|
return compileAggregateSet(projectId, query.op, query.inputs, source);
|
|
10741
11295
|
case "sort":
|
|
@@ -10800,8 +11354,11 @@ function compileAggregateWhere(projectId, inputQuery, predicate, source) {
|
|
|
10800
11354
|
args: [...input.args, ...predicate.args]
|
|
10801
11355
|
};
|
|
10802
11356
|
}
|
|
10803
|
-
function compileAggregateTraversal(projectId,
|
|
10804
|
-
const
|
|
11357
|
+
function compileAggregateTraversal(projectId, traversal, source) {
|
|
11358
|
+
const { linkId, direction } = traversal;
|
|
11359
|
+
const input = compileAggregateSource(projectId, traversal.input, source);
|
|
11360
|
+
const endpoints = linkEndpointObjectTypeIds(traversal);
|
|
11361
|
+
const sourceObjectTypeId = endpoints?.length === 1 ? endpoints[0] : undefined;
|
|
10805
11362
|
if (direction === "incoming" && sourceObjectTypeId !== undefined) {
|
|
10806
11363
|
return {
|
|
10807
11364
|
sql: `SELECT selected.project_id, selected.object_type_id, selected.primary_id, selected.properties
|
|
@@ -10822,7 +11379,7 @@ function compileAggregateTraversal(projectId, inputQuery, linkId, direction, sou
|
|
|
10822
11379
|
args: [...input.args, linkId, sourceObjectTypeId, projectId, sourceObjectTypeId]
|
|
10823
11380
|
};
|
|
10824
11381
|
}
|
|
10825
|
-
const relation = compileTraversalRelation(projectId, input,
|
|
11382
|
+
const relation = compileTraversalRelation(projectId, input, traversal, source);
|
|
10826
11383
|
return {
|
|
10827
11384
|
sql: `SELECT selected.project_id, selected.object_type_id, selected.primary_id, selected.properties ${relation.sql}`,
|
|
10828
11385
|
args: relation.args
|
|
@@ -10927,7 +11484,7 @@ function compilePredicate(predicate) {
|
|
|
10927
11484
|
case "constant":
|
|
10928
11485
|
return { sql: predicate.value ? "true" : "false", args: [] };
|
|
10929
11486
|
case "eq":
|
|
10930
|
-
return predicate.scalarKind === "decimal" ? compileDecimalEqualityPredicate(predicate.propertyId, "eq", predicate.value) : compileEqualityPredicate(predicate.propertyId, "eq", predicate.value);
|
|
11487
|
+
return predicate.scalarKind === "decimal" ? compileDecimalEqualityPredicate(predicate.propertyId, "eq", predicate.value) : compileEqualityPredicate(predicate.propertyId, "eq", predicate.value, predicate.scalarKind);
|
|
10931
11488
|
case "neq":
|
|
10932
11489
|
return predicate.scalarKind === "decimal" ? compileDecimalEqualityPredicate(predicate.propertyId, "neq", predicate.value) : compileEqualityPredicate(predicate.propertyId, "neq", predicate.value);
|
|
10933
11490
|
case "lt":
|
|
@@ -10953,7 +11510,7 @@ function compilePredicate(predicate) {
|
|
|
10953
11510
|
};
|
|
10954
11511
|
}
|
|
10955
11512
|
case "in":
|
|
10956
|
-
return predicate.scalarKind === "decimal" ? compileDecimalInPredicate(predicate.propertyId, predicate.values) : compileInPredicate(predicate.propertyId, predicate.values);
|
|
11513
|
+
return predicate.scalarKind === "decimal" ? compileDecimalInPredicate(predicate.propertyId, predicate.values) : compileInPredicate(predicate.propertyId, predicate.values, predicate.scalarKind);
|
|
10957
11514
|
case "exists": {
|
|
10958
11515
|
const sql = `jsonb_exists(properties, ?::text)`;
|
|
10959
11516
|
const item = { sql, args: [predicate.propertyId] };
|
|
@@ -11005,13 +11562,13 @@ function negatePredicate(predicate) {
|
|
|
11005
11562
|
args: predicate.args
|
|
11006
11563
|
};
|
|
11007
11564
|
}
|
|
11008
|
-
function compileEqualityPredicate(propertyId, op, value) {
|
|
11565
|
+
function compileEqualityPredicate(propertyId, op, value, scalarKind) {
|
|
11009
11566
|
if (value === null) {
|
|
11010
11567
|
const sql = `${jsonTypeExpression()} = 'null'`;
|
|
11011
11568
|
const item = { sql, args: [propertyId] };
|
|
11012
11569
|
return op === "eq" ? item : negatePredicate(item);
|
|
11013
11570
|
}
|
|
11014
|
-
if (typeof value === "string") {
|
|
11571
|
+
if (typeof value === "string" && !(op === "eq" && hasJsonValueIndex(scalarKind))) {
|
|
11015
11572
|
if (op === "eq") {
|
|
11016
11573
|
return {
|
|
11017
11574
|
sql: `(${jsonTypeExpression()} = 'string' AND ${jsonTextExpression()} = ?::text AND md5(${jsonTextExpression()}) = md5(?::text))`,
|
|
@@ -11034,12 +11591,13 @@ function compileEqualityPredicate(propertyId, op, value) {
|
|
|
11034
11591
|
args: [propertyId, propertyId, propertyId, jsonbValue(value)]
|
|
11035
11592
|
};
|
|
11036
11593
|
}
|
|
11037
|
-
function compileInPredicate(propertyId, values) {
|
|
11594
|
+
function compileInPredicate(propertyId, values, scalarKind) {
|
|
11038
11595
|
if (values.length === 0)
|
|
11039
11596
|
return { sql: "0 = 1", args: [] };
|
|
11040
11597
|
const nonNullValues = values.filter((value) => value !== null);
|
|
11041
|
-
const
|
|
11042
|
-
const
|
|
11598
|
+
const comparedAsText = (value) => typeof value === "string" && !hasJsonValueIndex(scalarKind);
|
|
11599
|
+
const stringValues = nonNullValues.filter(comparedAsText);
|
|
11600
|
+
const jsonbValues = nonNullValues.filter((value) => !comparedAsText(value));
|
|
11043
11601
|
const clauses = [];
|
|
11044
11602
|
const args = [];
|
|
11045
11603
|
if (values.length !== nonNullValues.length) {
|
|
@@ -11329,6 +11887,9 @@ function orderedValueExpression(scalarKind, propertyColumn = "properties") {
|
|
|
11329
11887
|
return `((${jsonTextExpression(propertyColumn)}) COLLATE "C")`;
|
|
11330
11888
|
return `NULLIF(${jsonValueExpression(propertyColumn)}, 'null'::jsonb)`;
|
|
11331
11889
|
}
|
|
11890
|
+
function hasJsonValueIndex(scalarKind) {
|
|
11891
|
+
return scalarKind === "date" || scalarKind === "timestamp" || scalarKind === "uuid";
|
|
11892
|
+
}
|
|
11332
11893
|
function isTextScalarKind(scalarKind) {
|
|
11333
11894
|
return scalarKind === "string" || scalarKind === "uuid";
|
|
11334
11895
|
}
|
|
@@ -11392,6 +11953,310 @@ function isPlainObject(value) {
|
|
|
11392
11953
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
11393
11954
|
}
|
|
11394
11955
|
|
|
11956
|
+
// src/objects/query-indexes.ts
|
|
11957
|
+
async function ensurePgObjectQueryIndexes(sql, schema, indexes) {
|
|
11958
|
+
const definitions = indexes.map((index) => ({
|
|
11959
|
+
table: index.kind === "incomingLinks" ? "links" : objectPartitionName(index.objectTypeId),
|
|
11960
|
+
expression: compileIndex(index),
|
|
11961
|
+
index
|
|
11962
|
+
}));
|
|
11963
|
+
if (indexes.some((index) => index.kind === "text")) {
|
|
11964
|
+
const [extension] = await sql`
|
|
11965
|
+
SELECT EXISTS (
|
|
11966
|
+
SELECT 1 FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace
|
|
11967
|
+
WHERE e.extname = 'pg_trgm' AND n.nspname = 'public'
|
|
11968
|
+
) AS ready`;
|
|
11969
|
+
if (!extension?.ready) {
|
|
11970
|
+
throw new Error("[SixbPg] Text query indexes require pg_trgm in schema public. Run CREATE EXTENSION pg_trgm WITH SCHEMA public before ensuring indexes.");
|
|
11971
|
+
}
|
|
11972
|
+
}
|
|
11973
|
+
const names = [];
|
|
11974
|
+
await withReservedPgConnection(sql, async (connection) => {
|
|
11975
|
+
let connected = true;
|
|
11976
|
+
try {
|
|
11977
|
+
await lockPgPreparation(connection, `${schema}.objects.query-indexes`);
|
|
11978
|
+
for (const objectTypeId of new Set(indexes.flatMap((index) => index.kind === "incomingLinks" ? [] : [index.objectTypeId]))) {
|
|
11979
|
+
await ensureObjectPartition(connection, schema, objectTypeId);
|
|
11980
|
+
}
|
|
11981
|
+
for (const definition of definitions) {
|
|
11982
|
+
names.push(await createQueryIndex(connection, schema, definition));
|
|
11983
|
+
}
|
|
11984
|
+
await ensureExpressionStatistics(connection, schema, indexes);
|
|
11985
|
+
} catch (error) {
|
|
11986
|
+
connected = !isConnectionLost(error);
|
|
11987
|
+
throw error;
|
|
11988
|
+
} finally {
|
|
11989
|
+
if (connected)
|
|
11990
|
+
await connection`SELECT pg_advisory_unlock(hashtextextended(${`${schema}.objects.query-indexes`}, 0))`;
|
|
11991
|
+
}
|
|
11992
|
+
});
|
|
11993
|
+
return names;
|
|
11994
|
+
}
|
|
11995
|
+
async function createQueryIndex(connection, schema, definition) {
|
|
11996
|
+
const { table } = definition;
|
|
11997
|
+
const name = `sixb_query_${createHash4("sha256").update(`${table}:${definition.expression}`).digest("hex").slice(0, 32)}`;
|
|
11998
|
+
await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${identifier(schema)}.${identifier(table)} ${definition.expression}`);
|
|
11999
|
+
const [index] = await connection`
|
|
12000
|
+
SELECT i.indisvalid AS valid FROM pg_index i
|
|
12001
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
12002
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
12003
|
+
WHERE n.nspname = ${schema} AND c.relname = ${name}`;
|
|
12004
|
+
if (!index?.valid)
|
|
12005
|
+
throw new Error(`[SixbPg] Query index ${schema}.${name} is invalid after an interrupted build. Drop that index concurrently and retry ensureObjectQueryIndexes().`);
|
|
12006
|
+
if (definition.index.kind === "sort") {
|
|
12007
|
+
await connection.unsafe(`COMMENT ON INDEX ${identifier(schema)}.${identifier(name)} IS ${literal(JSON.stringify(definition.index))}`);
|
|
12008
|
+
await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(`${name}_overflow`)} ON ${identifier(schema)}.${identifier(table)} (primary_id) WHERE project_id = ${literal(definition.index.projectId)} AND NOT (${sortIndexBound(definition.index)})`);
|
|
12009
|
+
const superseded = await connection`
|
|
12010
|
+
SELECT c.relname AS name FROM pg_index i
|
|
12011
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
12012
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
12013
|
+
WHERE n.nspname = ${schema} AND c.relname LIKE 'sixb_query_%' AND c.relname <> ${name}
|
|
12014
|
+
AND obj_description(c.oid, 'pg_class') = ${JSON.stringify(definition.index)}`;
|
|
12015
|
+
for (const previous of superseded) {
|
|
12016
|
+
for (const index of [previous.name, `${previous.name}_overflow`]) {
|
|
12017
|
+
await connection.unsafe(`DROP INDEX CONCURRENTLY IF EXISTS ${identifier(schema)}.${identifier(index)}`);
|
|
12018
|
+
}
|
|
12019
|
+
}
|
|
12020
|
+
}
|
|
12021
|
+
return name;
|
|
12022
|
+
}
|
|
12023
|
+
async function ensureExpressionStatistics(connection, schema, indexes) {
|
|
12024
|
+
const expressionsByType = new Map;
|
|
12025
|
+
for (const index of indexes) {
|
|
12026
|
+
if (index.kind === "incomingLinks")
|
|
12027
|
+
continue;
|
|
12028
|
+
const expressions = expressionsByType.get(index.objectTypeId) ?? new Set;
|
|
12029
|
+
for (const expression of statisticsExpressions(index))
|
|
12030
|
+
expressions.add(expression);
|
|
12031
|
+
expressionsByType.set(index.objectTypeId, expressions);
|
|
12032
|
+
}
|
|
12033
|
+
for (const [objectTypeId, expressions] of expressionsByType) {
|
|
12034
|
+
const partition = `${identifier(schema)}.${identifier(objectPartitionName(objectTypeId))}`;
|
|
12035
|
+
let statisticsChanged = false;
|
|
12036
|
+
for (const expression of expressions) {
|
|
12037
|
+
const name = `sixb_query_stats_${createHash4("sha256").update(`${objectTypeId}:${expression}`).digest("hex").slice(0, 24)}`;
|
|
12038
|
+
const [existing] = await connection`
|
|
12039
|
+
SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
|
|
12040
|
+
WHERE n.nspname = ${schema} AND s.stxname = ${name}`;
|
|
12041
|
+
if (existing)
|
|
12042
|
+
continue;
|
|
12043
|
+
await connection.unsafe(`CREATE STATISTICS ${identifier(schema)}.${identifier(name)} ON project_id, ${expression} FROM ${partition}`);
|
|
12044
|
+
statisticsChanged = true;
|
|
12045
|
+
}
|
|
12046
|
+
if (statisticsChanged)
|
|
12047
|
+
await connection.unsafe(`ANALYZE ${partition}`);
|
|
12048
|
+
}
|
|
12049
|
+
}
|
|
12050
|
+
function statisticsExpressions(index) {
|
|
12051
|
+
const equality = new Set(Object.keys(index.where ?? {}));
|
|
12052
|
+
if (index.kind === "filter")
|
|
12053
|
+
for (const property of index.properties)
|
|
12054
|
+
equality.add(property);
|
|
12055
|
+
if (index.kind === "sort")
|
|
12056
|
+
for (const property of index.equality ?? [])
|
|
12057
|
+
equality.add(property);
|
|
12058
|
+
const expressions = [...equality].map((property) => `jsonb_typeof(properties -> ${literal(property)}), (properties ->> ${literal(property)}), (md5(properties ->> ${literal(property)}))`);
|
|
12059
|
+
if (index.kind === "value")
|
|
12060
|
+
expressions.push(`(properties -> ${literal(index.propertyId)})`);
|
|
12061
|
+
if (index.kind === "text")
|
|
12062
|
+
expressions.push(`lower(coalesce(properties ->> ${literal(index.propertyId)}, ''))`);
|
|
12063
|
+
if (index.kind === "sort")
|
|
12064
|
+
for (const field of index.fields)
|
|
12065
|
+
expressions.push(sortValue(field));
|
|
12066
|
+
return expressions;
|
|
12067
|
+
}
|
|
12068
|
+
function compileIndex(index) {
|
|
12069
|
+
if (index.kind === "incomingLinks") {
|
|
12070
|
+
if (index.where)
|
|
12071
|
+
throw new Error("[SixbPg] Incoming link indexes do not accept object property conditions");
|
|
12072
|
+
return `(target_id, link_id, source_type_id, source_id) INCLUDE (project_id, target_type_id) WHERE project_id = ${literal(index.projectId)} AND target_type_id = ${literal(index.objectTypeId)}`;
|
|
12073
|
+
}
|
|
12074
|
+
const predicate = [`project_id = ${literal(index.projectId)}`];
|
|
12075
|
+
for (const [propertyId, value] of Object.entries(index.where ?? {}).sort(([a], [b]) => a.localeCompare(b))) {
|
|
12076
|
+
const property = `properties -> ${literal(propertyId)}`;
|
|
12077
|
+
if (value === null)
|
|
12078
|
+
predicate.push(`jsonb_typeof(${property}) = 'null'`);
|
|
12079
|
+
else if (typeof value === "string") {
|
|
12080
|
+
predicate.push(`jsonb_typeof(${property}) = 'string'`, `properties ->> ${literal(propertyId)} = ${literal(value)}`, `md5(properties ->> ${literal(propertyId)}) = md5(${literal(value)})`);
|
|
12081
|
+
} else {
|
|
12082
|
+
if (typeof value === "number" && !Number.isFinite(value)) {
|
|
12083
|
+
throw new Error("[SixbPg] Query index conditions require finite numbers");
|
|
12084
|
+
}
|
|
12085
|
+
predicate.push(`${property} = ${literal(JSON.stringify(value))}::jsonb`);
|
|
12086
|
+
}
|
|
12087
|
+
}
|
|
12088
|
+
if (index.kind === "text") {
|
|
12089
|
+
return `USING gin (lower(coalesce(properties ->> ${literal(index.propertyId)}, '')) public.gin_trgm_ops) WHERE ${predicate.join(" AND ")}`;
|
|
12090
|
+
}
|
|
12091
|
+
const fields = [];
|
|
12092
|
+
if (index.kind === "count") {
|
|
12093
|
+
fields.push("project_id", "object_type_id");
|
|
12094
|
+
} else if (index.kind === "value") {
|
|
12095
|
+
fields.push(`(properties -> ${literal(index.propertyId)})`);
|
|
12096
|
+
}
|
|
12097
|
+
if (index.kind === "filter") {
|
|
12098
|
+
if (!index.properties.length || index.properties.length > 8)
|
|
12099
|
+
throw new Error("[SixbPg] A filter index requires one to eight string properties");
|
|
12100
|
+
for (const propertyId of index.properties)
|
|
12101
|
+
fields.push(`(md5(properties ->> ${literal(propertyId)}))`);
|
|
12102
|
+
}
|
|
12103
|
+
if (index.kind === "sort") {
|
|
12104
|
+
predicate.push(sortIndexBound(index));
|
|
12105
|
+
if (index.fields.length === 0 || index.fields.length + (index.equality?.length ?? 0) > 8) {
|
|
12106
|
+
throw new Error("[SixbPg] A sort query index requires between one and eight fields");
|
|
12107
|
+
}
|
|
12108
|
+
for (const propertyId of index.equality ?? []) {
|
|
12109
|
+
fields.push(`(properties ->> ${literal(propertyId)})`);
|
|
12110
|
+
}
|
|
12111
|
+
for (const field of index.fields) {
|
|
12112
|
+
const property = `properties -> ${literal(field.propertyId)}`;
|
|
12113
|
+
fields.push(`(CASE WHEN jsonb_typeof(${property}) IS NULL OR jsonb_typeof(${property}) = 'null' THEN 1 ELSE 0 END)`, `${sortValue(field)} ${field.direction === "desc" ? "DESC" : "ASC"}`);
|
|
12114
|
+
}
|
|
12115
|
+
fields.push("primary_id");
|
|
12116
|
+
}
|
|
12117
|
+
return `(${fields.join(", ")}) WHERE ${predicate.join(" AND ")}`;
|
|
12118
|
+
}
|
|
12119
|
+
function sortValue(field) {
|
|
12120
|
+
const text = `(properties ->> ${literal(field.propertyId)})`;
|
|
12121
|
+
if (field.scalarKind === "decimal")
|
|
12122
|
+
return `(${text}::numeric)`;
|
|
12123
|
+
if (isTextScalarKind(field.scalarKind))
|
|
12124
|
+
return `(${text} COLLATE "C")`;
|
|
12125
|
+
return `(NULLIF(properties -> ${literal(field.propertyId)}, 'null'::jsonb))`;
|
|
12126
|
+
}
|
|
12127
|
+
function sortIndexBound(index) {
|
|
12128
|
+
const sizes = [
|
|
12129
|
+
"octet_length(primary_id)",
|
|
12130
|
+
...index.fields.map((field) => `coalesce(octet_length((properties -> ${literal(field.propertyId)})::text), 0)`),
|
|
12131
|
+
...(index.equality ?? []).map((property) => `coalesce(octet_length(properties ->> ${literal(property)}), 0)`)
|
|
12132
|
+
];
|
|
12133
|
+
return `(${sizes.join(" + ")}) <= 1800`;
|
|
12134
|
+
}
|
|
12135
|
+
var SORT_INDEX_CATALOG_TTL_MS = 60000;
|
|
12136
|
+
|
|
12137
|
+
class PgSortIndexCatalog {
|
|
12138
|
+
#read;
|
|
12139
|
+
indexes(sql) {
|
|
12140
|
+
const now = Date.now();
|
|
12141
|
+
if (this.#read && now - this.#read.at < SORT_INDEX_CATALOG_TTL_MS)
|
|
12142
|
+
return this.#read.indexes;
|
|
12143
|
+
const read = { at: now, indexes: readPgSortIndexes(sql) };
|
|
12144
|
+
this.#read = read;
|
|
12145
|
+
read.indexes.catch(() => {
|
|
12146
|
+
if (this.#read === read)
|
|
12147
|
+
this.#read = undefined;
|
|
12148
|
+
});
|
|
12149
|
+
return read.indexes;
|
|
12150
|
+
}
|
|
12151
|
+
invalidate() {
|
|
12152
|
+
this.#read = undefined;
|
|
12153
|
+
}
|
|
12154
|
+
}
|
|
12155
|
+
async function readPgSortIndexes(sql) {
|
|
12156
|
+
const rows = await sql`
|
|
12157
|
+
SELECT d.description AS definition
|
|
12158
|
+
FROM pg_index i
|
|
12159
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
12160
|
+
JOIN pg_description d
|
|
12161
|
+
ON d.objoid = c.oid AND d.classoid = 'pg_class'::regclass AND d.objsubid = 0
|
|
12162
|
+
WHERE i.indrelid IN (SELECT inhrelid FROM pg_inherits WHERE inhparent = 'objects'::regclass)
|
|
12163
|
+
AND i.indisvalid AND c.relname LIKE 'sixb_query_%'
|
|
12164
|
+
`;
|
|
12165
|
+
const indexes = [];
|
|
12166
|
+
for (const row of rows) {
|
|
12167
|
+
let candidate;
|
|
12168
|
+
try {
|
|
12169
|
+
candidate = JSON.parse(row.definition);
|
|
12170
|
+
} catch {
|
|
12171
|
+
continue;
|
|
12172
|
+
}
|
|
12173
|
+
if (!candidate || typeof candidate !== "object")
|
|
12174
|
+
continue;
|
|
12175
|
+
if (candidate.kind === "sort")
|
|
12176
|
+
indexes.push(candidate);
|
|
12177
|
+
}
|
|
12178
|
+
return indexes;
|
|
12179
|
+
}
|
|
12180
|
+
async function findPgSortIndexBound(sql, catalog, projectId, query) {
|
|
12181
|
+
while (query.kind === "expand")
|
|
12182
|
+
query = query.input;
|
|
12183
|
+
if (query.kind !== "page" && query.kind !== "limit")
|
|
12184
|
+
return;
|
|
12185
|
+
let input = query.input;
|
|
12186
|
+
let fields;
|
|
12187
|
+
const equality = new Set;
|
|
12188
|
+
while (input.kind === "sort" || input.kind === "filter" || input.kind === "text") {
|
|
12189
|
+
if (input.kind === "sort") {
|
|
12190
|
+
if (fields)
|
|
12191
|
+
return;
|
|
12192
|
+
fields = input.fields;
|
|
12193
|
+
}
|
|
12194
|
+
if (input.kind === "filter") {
|
|
12195
|
+
const inspect = (predicate) => {
|
|
12196
|
+
if (predicate.op === "and")
|
|
12197
|
+
predicate.items.forEach(inspect);
|
|
12198
|
+
else if (predicate.op === "eq" && typeof predicate.value === "string" && !hasJsonValueIndex(predicate.scalarKind))
|
|
12199
|
+
equality.add(predicate.propertyId);
|
|
12200
|
+
else if (predicate.op === "in" && predicate.values.length === 1 && typeof predicate.values[0] === "string" && !hasJsonValueIndex(predicate.scalarKind))
|
|
12201
|
+
equality.add(predicate.propertyId);
|
|
12202
|
+
};
|
|
12203
|
+
inspect(input.predicate);
|
|
12204
|
+
}
|
|
12205
|
+
input = input.input;
|
|
12206
|
+
}
|
|
12207
|
+
let objectTypeId;
|
|
12208
|
+
if (input.kind === "start" && !input.includeSubtypes) {
|
|
12209
|
+
objectTypeId = input.objectTypeId;
|
|
12210
|
+
} else if (input.kind === "traverse") {
|
|
12211
|
+
const endpoints = linkEndpointObjectTypeIds2(input);
|
|
12212
|
+
objectTypeId = endpoints?.length === 1 ? endpoints[0] : undefined;
|
|
12213
|
+
}
|
|
12214
|
+
if (!objectTypeId || !fields?.length || fields.some((field) => field.kind !== "property"))
|
|
12215
|
+
return;
|
|
12216
|
+
const matches = [];
|
|
12217
|
+
for (const index of await catalog.indexes(sql)) {
|
|
12218
|
+
if (index.projectId !== projectId || index.objectTypeId !== objectTypeId || index.where || !Array.isArray(index.fields) || index.fields.length !== fields.length)
|
|
12219
|
+
continue;
|
|
12220
|
+
if (index.equality !== undefined && (!Array.isArray(index.equality) || !index.equality.every((property) => typeof property === "string" && equality.has(property))))
|
|
12221
|
+
continue;
|
|
12222
|
+
if (!index.fields.every((field, i) => field && typeof field.propertyId === "string" && fields[i]?.kind === "property" && fields[i].propertyId === field.propertyId && (field.direction ?? "asc") === (fields[i].direction ?? "asc") && field.scalarKind === "decimal" === (fields[i].scalarKind === "decimal") && isTextScalarKind(field.scalarKind) === isTextScalarKind(fields[i].scalarKind)))
|
|
12223
|
+
continue;
|
|
12224
|
+
matches.push(index);
|
|
12225
|
+
}
|
|
12226
|
+
matches.sort((a, b) => (b.equality?.length ?? 0) - (a.equality?.length ?? 0));
|
|
12227
|
+
return matches[0] ? sortIndexBound(matches[0]) : undefined;
|
|
12228
|
+
}
|
|
12229
|
+
// src/objects/storage.ts
|
|
12230
|
+
import { hasVectorProfile as hasVectorProfile4 } from "@sixb/core/internal/query";
|
|
12231
|
+
|
|
12232
|
+
// src/objects/aggregate-admission.ts
|
|
12233
|
+
class PgAggregateAdmission {
|
|
12234
|
+
concurrency;
|
|
12235
|
+
active = 0;
|
|
12236
|
+
waiting = [];
|
|
12237
|
+
constructor(concurrency = 1) {
|
|
12238
|
+
this.concurrency = concurrency;
|
|
12239
|
+
}
|
|
12240
|
+
async run(compute) {
|
|
12241
|
+
if (this.active >= this.concurrency)
|
|
12242
|
+
await new Promise((resolve) => this.waiting.push(resolve));
|
|
12243
|
+
else
|
|
12244
|
+
this.active++;
|
|
12245
|
+
try {
|
|
12246
|
+
return await compute();
|
|
12247
|
+
} finally {
|
|
12248
|
+
const next = this.waiting.shift();
|
|
12249
|
+
if (next)
|
|
12250
|
+
next();
|
|
12251
|
+
else
|
|
12252
|
+
this.active--;
|
|
12253
|
+
}
|
|
12254
|
+
}
|
|
12255
|
+
}
|
|
12256
|
+
|
|
12257
|
+
// src/objects/conditional-reader.ts
|
|
12258
|
+
import { hasVectorProfile as hasVectorProfile3 } from "@sixb/core/internal/query";
|
|
12259
|
+
|
|
11395
12260
|
// src/objects/read-scope.ts
|
|
11396
12261
|
function compilePgSelectedObjectReadSource(projectId, scope, maxTraversalFacts) {
|
|
11397
12262
|
const scopeJson = JSON.stringify({
|
|
@@ -11585,442 +12450,183 @@ function compileSelectedReadScopeCte(scopeJson, projectId) {
|
|
|
11585
12450
|
FROM _sixb_scope_reachable AS parent
|
|
11586
12451
|
JOIN _sixb_scope_link_selections AS selection
|
|
11587
12452
|
ON selection.parent_node_id = parent.node_id
|
|
11588
|
-
AND selection.source_object_type_id = parent.object_type_id
|
|
11589
|
-
JOIN links AS edge
|
|
11590
|
-
ON edge.project_id = parent.project_id
|
|
11591
|
-
AND edge.source_type_id = parent.object_type_id
|
|
11592
|
-
AND edge.source_id = parent.primary_id
|
|
11593
|
-
AND edge.link_id = selection.link_id
|
|
11594
|
-
AND edge.target_type_id = selection.target_object_type_id
|
|
11595
|
-
JOIN objects AS target
|
|
11596
|
-
ON target.project_id = edge.project_id
|
|
11597
|
-
AND target.object_type_id = edge.target_type_id
|
|
11598
|
-
AND target.primary_id = edge.target_id
|
|
11599
|
-
),
|
|
11600
|
-
_sixb_scope_object_permissions(project_id, object_type_id, primary_id, property_id) AS (
|
|
11601
|
-
SELECT DISTINCT
|
|
11602
|
-
reachable.project_id,
|
|
11603
|
-
reachable.object_type_id,
|
|
11604
|
-
reachable.primary_id,
|
|
11605
|
-
selected_property.value
|
|
11606
|
-
FROM _sixb_scope_reachable AS reachable
|
|
11607
|
-
JOIN _sixb_scope_object_selections AS selection
|
|
11608
|
-
ON selection.node_id = reachable.node_id
|
|
11609
|
-
AND selection.object_type_id = reachable.object_type_id
|
|
11610
|
-
CROSS JOIN LATERAL jsonb_array_elements_text(selection.property_ids)
|
|
11611
|
-
AS selected_property(value)
|
|
11612
|
-
),
|
|
11613
|
-
_sixb_scope_object_ids(project_id, object_type_id, primary_id) AS (
|
|
11614
|
-
SELECT DISTINCT project_id, object_type_id, primary_id
|
|
11615
|
-
FROM _sixb_scope_reachable
|
|
11616
|
-
),
|
|
11617
|
-
_sixb_scope_object_grants AS (
|
|
11618
|
-
SELECT reachable.project_id, reachable.object_type_id, reachable.primary_id,
|
|
11619
|
-
array_agg(DISTINCT selected_property.value) FILTER (WHERE selected_property.value IS NOT NULL) AS property_ids
|
|
11620
|
-
FROM _sixb_scope_reachable AS reachable
|
|
11621
|
-
LEFT JOIN _sixb_scope_object_selections AS selection
|
|
11622
|
-
ON selection.node_id = reachable.node_id
|
|
11623
|
-
AND selection.object_type_id = reachable.object_type_id
|
|
11624
|
-
LEFT JOIN LATERAL jsonb_array_elements_text(selection.property_ids) AS selected_property(value) ON true
|
|
11625
|
-
GROUP BY reachable.project_id, reachable.object_type_id, reachable.primary_id
|
|
11626
|
-
),
|
|
11627
|
-
_sixb_scope_objects AS (
|
|
11628
|
-
SELECT stored.project_id, stored.object_type_id, stored.primary_id,
|
|
11629
|
-
COALESCE((
|
|
11630
|
-
SELECT jsonb_object_agg(property.key, property.value)
|
|
11631
|
-
FROM jsonb_each(stored.properties) AS property(key, value)
|
|
11632
|
-
WHERE property.key = ANY(grants.property_ids)
|
|
11633
|
-
), '{}'::jsonb) AS properties,
|
|
11634
|
-
stored.created_at, stored.updated_at, stored.version, stored.last_commit_id
|
|
11635
|
-
FROM _sixb_scope_object_grants AS grants
|
|
11636
|
-
JOIN objects AS stored
|
|
11637
|
-
ON stored.project_id = grants.project_id
|
|
11638
|
-
AND stored.object_type_id = grants.object_type_id
|
|
11639
|
-
AND stored.primary_id = grants.primary_id
|
|
11640
|
-
),
|
|
11641
|
-
_sixb_scope_reached_links(
|
|
11642
|
-
project_id,
|
|
11643
|
-
step_id,
|
|
11644
|
-
node_id,
|
|
11645
|
-
parent_node_id,
|
|
11646
|
-
source_type_id,
|
|
11647
|
-
source_id,
|
|
11648
|
-
link_id,
|
|
11649
|
-
target_type_id,
|
|
11650
|
-
target_id,
|
|
11651
|
-
property_ids
|
|
11652
|
-
) AS (
|
|
11653
|
-
SELECT
|
|
11654
|
-
edge.project_id,
|
|
11655
|
-
selection.step_id,
|
|
11656
|
-
selection.node_id,
|
|
11657
|
-
selection.parent_node_id,
|
|
11658
|
-
edge.source_type_id,
|
|
11659
|
-
edge.source_id,
|
|
11660
|
-
edge.link_id,
|
|
11661
|
-
edge.target_type_id,
|
|
11662
|
-
edge.target_id,
|
|
11663
|
-
selection.property_ids
|
|
11664
|
-
FROM _sixb_scope_reachable AS parent
|
|
11665
|
-
JOIN _sixb_scope_link_selections AS selection
|
|
11666
|
-
ON selection.parent_node_id = parent.node_id
|
|
11667
|
-
AND selection.source_object_type_id = parent.object_type_id
|
|
11668
|
-
JOIN links AS edge
|
|
11669
|
-
ON edge.project_id = parent.project_id
|
|
11670
|
-
AND edge.source_type_id = parent.object_type_id
|
|
11671
|
-
AND edge.source_id = parent.primary_id
|
|
11672
|
-
AND edge.link_id = selection.link_id
|
|
11673
|
-
AND edge.target_type_id = selection.target_object_type_id
|
|
11674
|
-
JOIN objects AS target
|
|
11675
|
-
ON target.project_id = edge.project_id
|
|
11676
|
-
AND target.object_type_id = edge.target_type_id
|
|
11677
|
-
AND target.primary_id = edge.target_id
|
|
11678
|
-
),
|
|
11679
|
-
_sixb_scope_link_permissions(
|
|
11680
|
-
project_id,
|
|
11681
|
-
source_type_id,
|
|
11682
|
-
source_id,
|
|
11683
|
-
link_id,
|
|
11684
|
-
target_type_id,
|
|
11685
|
-
target_id,
|
|
11686
|
-
property_id
|
|
11687
|
-
) AS (
|
|
11688
|
-
SELECT DISTINCT
|
|
11689
|
-
reached.project_id,
|
|
11690
|
-
reached.source_type_id,
|
|
11691
|
-
reached.source_id,
|
|
11692
|
-
reached.link_id,
|
|
11693
|
-
reached.target_type_id,
|
|
11694
|
-
reached.target_id,
|
|
11695
|
-
selected_property.value
|
|
11696
|
-
FROM _sixb_scope_reached_links AS reached
|
|
11697
|
-
CROSS JOIN LATERAL jsonb_array_elements_text(reached.property_ids)
|
|
11698
|
-
AS selected_property(value)
|
|
11699
|
-
),
|
|
11700
|
-
_sixb_scope_link_ids(
|
|
11701
|
-
project_id,
|
|
11702
|
-
source_type_id,
|
|
11703
|
-
source_id,
|
|
11704
|
-
link_id,
|
|
11705
|
-
target_type_id,
|
|
11706
|
-
target_id
|
|
11707
|
-
) AS (
|
|
11708
|
-
SELECT DISTINCT
|
|
11709
|
-
project_id,
|
|
11710
|
-
source_type_id,
|
|
11711
|
-
source_id,
|
|
11712
|
-
link_id,
|
|
11713
|
-
target_type_id,
|
|
11714
|
-
target_id
|
|
11715
|
-
FROM _sixb_scope_reached_links
|
|
11716
|
-
),
|
|
11717
|
-
_sixb_scope_links AS (
|
|
11718
|
-
SELECT
|
|
11719
|
-
stored.project_id,
|
|
11720
|
-
stored.source_type_id,
|
|
11721
|
-
stored.source_id,
|
|
11722
|
-
stored.link_id,
|
|
11723
|
-
stored.target_type_id,
|
|
11724
|
-
stored.target_id,
|
|
11725
|
-
CASE
|
|
11726
|
-
WHEN stored.properties IS NULL THEN NULL
|
|
11727
|
-
ELSE NULLIF(
|
|
11728
|
-
(
|
|
11729
|
-
SELECT jsonb_object_agg(property.key, property.value)
|
|
11730
|
-
FROM jsonb_each(stored.properties) AS property(key, value)
|
|
11731
|
-
JOIN _sixb_scope_link_permissions AS permission
|
|
11732
|
-
ON permission.project_id = stored.project_id
|
|
11733
|
-
AND permission.source_type_id = stored.source_type_id
|
|
11734
|
-
AND permission.source_id = stored.source_id
|
|
11735
|
-
AND permission.link_id = stored.link_id
|
|
11736
|
-
AND permission.target_type_id = stored.target_type_id
|
|
11737
|
-
AND permission.target_id = stored.target_id
|
|
11738
|
-
AND permission.property_id = property.key
|
|
11739
|
-
),
|
|
11740
|
-
'{}'::jsonb
|
|
11741
|
-
)
|
|
11742
|
-
END AS properties,
|
|
11743
|
-
stored.created_at,
|
|
11744
|
-
stored.updated_at,
|
|
11745
|
-
stored.last_commit_id
|
|
11746
|
-
FROM links AS stored
|
|
11747
|
-
JOIN _sixb_scope_link_ids AS visible
|
|
11748
|
-
ON visible.project_id = stored.project_id
|
|
11749
|
-
AND visible.source_type_id = stored.source_type_id
|
|
11750
|
-
AND visible.source_id = stored.source_id
|
|
11751
|
-
AND visible.link_id = stored.link_id
|
|
11752
|
-
AND visible.target_type_id = stored.target_type_id
|
|
11753
|
-
AND visible.target_id = stored.target_id
|
|
11754
|
-
)
|
|
11755
|
-
`,
|
|
11756
|
-
args: [scopeJson, projectId]
|
|
11757
|
-
};
|
|
11758
|
-
}
|
|
11759
|
-
|
|
11760
|
-
// src/objects/reader.ts
|
|
11761
|
-
import { ObjectQueryExecutionError as ObjectQueryExecutionError2 } from "@sixb/core";
|
|
11762
|
-
import { hasVectorProfile as hasVectorProfile2 } from "@sixb/core/internal/query";
|
|
11763
|
-
import { linkBatchKey, objectBatchKey } from "@sixb/core/storage";
|
|
11764
|
-
|
|
11765
|
-
// src/objects/query-indexes.ts
|
|
11766
|
-
import { createHash as createHash3 } from "crypto";
|
|
11767
|
-
|
|
11768
|
-
// src/objects/preparation-lock.ts
|
|
11769
|
-
async function lockPgPreparation(sql, key) {
|
|
11770
|
-
let delay = 25;
|
|
11771
|
-
while (true) {
|
|
11772
|
-
const [row] = await sql`
|
|
11773
|
-
SELECT pg_try_advisory_lock(hashtextextended(${key}, 0)) AS acquired`;
|
|
11774
|
-
if (row?.acquired)
|
|
11775
|
-
return;
|
|
11776
|
-
await Bun.sleep(delay);
|
|
11777
|
-
delay = Math.min(delay * 2, 500);
|
|
11778
|
-
}
|
|
11779
|
-
}
|
|
11780
|
-
|
|
11781
|
-
// src/objects/query-indexes.ts
|
|
11782
|
-
async function ensurePgObjectQueryIndexes(sql, schema, indexes) {
|
|
11783
|
-
const definitions = indexes.map((index) => ({
|
|
11784
|
-
table: index.kind === "incomingLinks" ? "links" : "objects",
|
|
11785
|
-
expression: compileIndex(index),
|
|
11786
|
-
index
|
|
11787
|
-
}));
|
|
11788
|
-
if (indexes.some((index) => index.kind === "text")) {
|
|
11789
|
-
const [extension] = await sql`
|
|
11790
|
-
SELECT EXISTS (
|
|
11791
|
-
SELECT 1 FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace
|
|
11792
|
-
WHERE e.extname = 'pg_trgm' AND n.nspname = 'public'
|
|
11793
|
-
) AS ready`;
|
|
11794
|
-
if (!extension?.ready) {
|
|
11795
|
-
throw new Error("[SixbPg] Text query indexes require pg_trgm in schema public. Run CREATE EXTENSION pg_trgm WITH SCHEMA public before ensuring indexes.");
|
|
11796
|
-
}
|
|
11797
|
-
}
|
|
11798
|
-
const names = [];
|
|
11799
|
-
await withReservedPgConnection(sql, async (connection) => {
|
|
11800
|
-
let connected = true;
|
|
11801
|
-
try {
|
|
11802
|
-
await lockPgPreparation(connection, `${schema}.objects.query-indexes`);
|
|
11803
|
-
for (const definition of definitions) {
|
|
11804
|
-
names.push(await createQueryIndex(connection, schema, definition));
|
|
11805
|
-
}
|
|
11806
|
-
await ensureExpressionStatistics(connection, schema, indexes);
|
|
11807
|
-
} catch (error) {
|
|
11808
|
-
connected = !isConnectionLost(error);
|
|
11809
|
-
throw error;
|
|
11810
|
-
} finally {
|
|
11811
|
-
if (connected)
|
|
11812
|
-
await connection`SELECT pg_advisory_unlock(hashtextextended(${`${schema}.objects.query-indexes`}, 0))`;
|
|
11813
|
-
}
|
|
11814
|
-
});
|
|
11815
|
-
return names;
|
|
11816
|
-
}
|
|
11817
|
-
async function createQueryIndex(connection, schema, definition) {
|
|
11818
|
-
const name = `sixb_query_${createHash3("sha256").update(definition.table === "objects" ? definition.expression : `${definition.table}:${definition.expression}`).digest("hex").slice(0, 32)}`;
|
|
11819
|
-
await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${identifier(schema)}.${identifier(definition.table)} ${definition.expression}`);
|
|
11820
|
-
const [index] = await connection`
|
|
11821
|
-
SELECT i.indisvalid AS valid FROM pg_index i
|
|
11822
|
-
JOIN pg_class c ON c.oid = i.indexrelid
|
|
11823
|
-
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
11824
|
-
WHERE n.nspname = ${schema} AND c.relname = ${name}`;
|
|
11825
|
-
if (!index?.valid)
|
|
11826
|
-
throw new Error(`[SixbPg] Query index ${schema}.${name} is invalid after an interrupted build. Drop that index concurrently and retry ensureObjectQueryIndexes().`);
|
|
11827
|
-
if (definition.index.kind === "sort") {
|
|
11828
|
-
await connection.unsafe(`COMMENT ON INDEX ${identifier(schema)}.${identifier(name)} IS ${literal(JSON.stringify(definition.index))}`);
|
|
11829
|
-
const scope = `project_id = ${literal(definition.index.projectId)} AND object_type_id = ${literal(definition.index.objectTypeId)}`;
|
|
11830
|
-
await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(`${name}_overflow`)} ON ${identifier(schema)}.objects (primary_id) WHERE ${scope} AND NOT (${sortIndexBound(definition.index)})`);
|
|
11831
|
-
const superseded = await connection`
|
|
11832
|
-
SELECT c.relname AS name FROM pg_index i
|
|
11833
|
-
JOIN pg_class c ON c.oid = i.indexrelid
|
|
11834
|
-
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
11835
|
-
WHERE n.nspname = ${schema} AND c.relname LIKE 'sixb_query_%' AND c.relname <> ${name}
|
|
11836
|
-
AND obj_description(c.oid, 'pg_class') = ${JSON.stringify(definition.index)}`;
|
|
11837
|
-
for (const previous of superseded) {
|
|
11838
|
-
for (const index of [previous.name, `${previous.name}_overflow`]) {
|
|
11839
|
-
await connection.unsafe(`DROP INDEX CONCURRENTLY IF EXISTS ${identifier(schema)}.${identifier(index)}`);
|
|
11840
|
-
}
|
|
11841
|
-
}
|
|
11842
|
-
}
|
|
11843
|
-
return name;
|
|
11844
|
-
}
|
|
11845
|
-
async function ensureExpressionStatistics(connection, schema, indexes) {
|
|
11846
|
-
let statisticsChanged = false;
|
|
11847
|
-
const expressions = new Set(indexes.flatMap(statisticsExpressions));
|
|
11848
|
-
for (const expression of expressions) {
|
|
11849
|
-
const name = `sixb_query_stats_${createHash3("sha256").update(expression).digest("hex").slice(0, 24)}`;
|
|
11850
|
-
const [existing] = await connection`
|
|
11851
|
-
SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
|
|
11852
|
-
WHERE n.nspname = ${schema} AND s.stxname = ${name}`;
|
|
11853
|
-
if (existing)
|
|
11854
|
-
continue;
|
|
11855
|
-
await connection.unsafe(`CREATE STATISTICS ${identifier(schema)}.${identifier(name)} ON project_id, object_type_id, ${expression} FROM ${identifier(schema)}.objects`);
|
|
11856
|
-
statisticsChanged = true;
|
|
11857
|
-
}
|
|
11858
|
-
if (statisticsChanged)
|
|
11859
|
-
await connection.unsafe(`ANALYZE ${identifier(schema)}.objects`);
|
|
11860
|
-
}
|
|
11861
|
-
function statisticsExpressions(index) {
|
|
11862
|
-
if (index.kind === "incomingLinks")
|
|
11863
|
-
return [];
|
|
11864
|
-
const equality = new Set(Object.keys(index.where ?? {}));
|
|
11865
|
-
if (index.kind === "filter")
|
|
11866
|
-
for (const property of index.properties)
|
|
11867
|
-
equality.add(property);
|
|
11868
|
-
if (index.kind === "sort")
|
|
11869
|
-
for (const property of index.equality ?? [])
|
|
11870
|
-
equality.add(property);
|
|
11871
|
-
const expressions = [...equality].map((property) => `jsonb_typeof(properties -> ${literal(property)}), (properties ->> ${literal(property)}), (md5(properties ->> ${literal(property)}))`);
|
|
11872
|
-
if (index.kind === "value")
|
|
11873
|
-
expressions.push(`(properties -> ${literal(index.propertyId)})`);
|
|
11874
|
-
if (index.kind === "text")
|
|
11875
|
-
expressions.push(`lower(coalesce(properties ->> ${literal(index.propertyId)}, ''))`);
|
|
11876
|
-
if (index.kind === "sort")
|
|
11877
|
-
for (const field of index.fields)
|
|
11878
|
-
expressions.push(sortValue(field));
|
|
11879
|
-
return expressions;
|
|
11880
|
-
}
|
|
11881
|
-
function compileIndex(index) {
|
|
11882
|
-
if (index.kind === "incomingLinks") {
|
|
11883
|
-
if (index.where)
|
|
11884
|
-
throw new Error("[SixbPg] Incoming link indexes do not accept object property conditions");
|
|
11885
|
-
return `(target_id, link_id, source_type_id, source_id) INCLUDE (project_id, target_type_id) WHERE project_id = ${literal(index.projectId)} AND target_type_id = ${literal(index.objectTypeId)}`;
|
|
11886
|
-
}
|
|
11887
|
-
const predicate = [
|
|
11888
|
-
`project_id = ${literal(index.projectId)}`,
|
|
11889
|
-
`object_type_id = ${literal(index.objectTypeId)}`
|
|
11890
|
-
];
|
|
11891
|
-
for (const [propertyId, value] of Object.entries(index.where ?? {}).sort(([a], [b]) => a.localeCompare(b))) {
|
|
11892
|
-
const property = `properties -> ${literal(propertyId)}`;
|
|
11893
|
-
if (value === null)
|
|
11894
|
-
predicate.push(`jsonb_typeof(${property}) = 'null'`);
|
|
11895
|
-
else if (typeof value === "string") {
|
|
11896
|
-
predicate.push(`jsonb_typeof(${property}) = 'string'`, `properties ->> ${literal(propertyId)} = ${literal(value)}`, `md5(properties ->> ${literal(propertyId)}) = md5(${literal(value)})`);
|
|
11897
|
-
} else {
|
|
11898
|
-
if (typeof value === "number" && !Number.isFinite(value)) {
|
|
11899
|
-
throw new Error("[SixbPg] Query index conditions require finite numbers");
|
|
11900
|
-
}
|
|
11901
|
-
predicate.push(`${property} = ${literal(JSON.stringify(value))}::jsonb`);
|
|
11902
|
-
}
|
|
11903
|
-
}
|
|
11904
|
-
if (index.kind === "text") {
|
|
11905
|
-
return `USING gin (lower(coalesce(properties ->> ${literal(index.propertyId)}, '')) public.gin_trgm_ops) WHERE ${predicate.join(" AND ")}`;
|
|
11906
|
-
}
|
|
11907
|
-
const fields = [];
|
|
11908
|
-
if (index.kind === "count") {
|
|
11909
|
-
fields.push("project_id", "object_type_id");
|
|
11910
|
-
} else if (index.kind === "value") {
|
|
11911
|
-
fields.push(`(properties -> ${literal(index.propertyId)})`);
|
|
11912
|
-
}
|
|
11913
|
-
if (index.kind === "filter") {
|
|
11914
|
-
if (!index.properties.length || index.properties.length > 8)
|
|
11915
|
-
throw new Error("[SixbPg] A filter index requires one to eight string properties");
|
|
11916
|
-
for (const propertyId of index.properties)
|
|
11917
|
-
fields.push(`(md5(properties ->> ${literal(propertyId)}))`);
|
|
11918
|
-
}
|
|
11919
|
-
if (index.kind === "sort") {
|
|
11920
|
-
predicate.push(sortIndexBound(index));
|
|
11921
|
-
if (index.fields.length === 0 || index.fields.length + (index.equality?.length ?? 0) > 8) {
|
|
11922
|
-
throw new Error("[SixbPg] A sort query index requires between one and eight fields");
|
|
11923
|
-
}
|
|
11924
|
-
for (const propertyId of index.equality ?? []) {
|
|
11925
|
-
fields.push(`(properties ->> ${literal(propertyId)})`);
|
|
11926
|
-
}
|
|
11927
|
-
for (const field of index.fields) {
|
|
11928
|
-
const property = `properties -> ${literal(field.propertyId)}`;
|
|
11929
|
-
fields.push(`(CASE WHEN jsonb_typeof(${property}) IS NULL OR jsonb_typeof(${property}) = 'null' THEN 1 ELSE 0 END)`, `${sortValue(field)} ${field.direction === "desc" ? "DESC" : "ASC"}`);
|
|
11930
|
-
}
|
|
11931
|
-
fields.push("primary_id");
|
|
11932
|
-
}
|
|
11933
|
-
return `(${fields.join(", ")}) WHERE ${predicate.join(" AND ")}`;
|
|
11934
|
-
}
|
|
11935
|
-
function sortValue(field) {
|
|
11936
|
-
const text = `(properties ->> ${literal(field.propertyId)})`;
|
|
11937
|
-
if (field.scalarKind === "decimal")
|
|
11938
|
-
return `(${text}::numeric)`;
|
|
11939
|
-
if (isTextScalarKind(field.scalarKind))
|
|
11940
|
-
return `(${text} COLLATE "C")`;
|
|
11941
|
-
return `(NULLIF(properties -> ${literal(field.propertyId)}, 'null'::jsonb))`;
|
|
11942
|
-
}
|
|
11943
|
-
function literal(value) {
|
|
11944
|
-
if (value.includes("\x00"))
|
|
11945
|
-
throw new Error("[SixbPg] Query index names and values cannot contain NUL");
|
|
11946
|
-
return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`;
|
|
11947
|
-
}
|
|
11948
|
-
function identifier(value) {
|
|
11949
|
-
return `"${value.replace(/"/g, '""')}"`;
|
|
11950
|
-
}
|
|
11951
|
-
function sortIndexBound(index) {
|
|
11952
|
-
const sizes = [
|
|
11953
|
-
"octet_length(primary_id)",
|
|
11954
|
-
...index.fields.map((field) => `coalesce(octet_length((properties -> ${literal(field.propertyId)})::text), 0)`),
|
|
11955
|
-
...(index.equality ?? []).map((property) => `coalesce(octet_length(properties ->> ${literal(property)}), 0)`)
|
|
11956
|
-
];
|
|
11957
|
-
return `(${sizes.join(" + ")}) <= 1800`;
|
|
11958
|
-
}
|
|
11959
|
-
async function findPgSortIndexBound(sql, projectId, query) {
|
|
11960
|
-
while (query.kind === "expand")
|
|
11961
|
-
query = query.input;
|
|
11962
|
-
if (query.kind !== "page" && query.kind !== "limit")
|
|
11963
|
-
return;
|
|
11964
|
-
let input = query.input;
|
|
11965
|
-
let fields;
|
|
11966
|
-
const equality = new Set;
|
|
11967
|
-
while (input.kind === "sort" || input.kind === "filter" || input.kind === "text") {
|
|
11968
|
-
if (input.kind === "sort") {
|
|
11969
|
-
if (fields)
|
|
11970
|
-
return;
|
|
11971
|
-
fields = input.fields;
|
|
11972
|
-
}
|
|
11973
|
-
if (input.kind === "filter") {
|
|
11974
|
-
const inspect = (predicate) => {
|
|
11975
|
-
if (predicate.op === "and")
|
|
11976
|
-
predicate.items.forEach(inspect);
|
|
11977
|
-
else if (predicate.op === "eq" && typeof predicate.value === "string")
|
|
11978
|
-
equality.add(predicate.propertyId);
|
|
11979
|
-
else if (predicate.op === "in" && predicate.values.length === 1 && typeof predicate.values[0] === "string")
|
|
11980
|
-
equality.add(predicate.propertyId);
|
|
11981
|
-
};
|
|
11982
|
-
inspect(input.predicate);
|
|
11983
|
-
}
|
|
11984
|
-
input = input.input;
|
|
11985
|
-
}
|
|
11986
|
-
let objectTypeId;
|
|
11987
|
-
if (input.kind === "start" && !input.includeSubtypes) {
|
|
11988
|
-
objectTypeId = input.objectTypeId;
|
|
11989
|
-
} else if (input.kind === "traverse" && input.direction === "incoming") {
|
|
11990
|
-
objectTypeId = input.sourceObjectTypeId;
|
|
11991
|
-
}
|
|
11992
|
-
if (!objectTypeId || !fields?.length || fields.some((field) => field.kind !== "property"))
|
|
11993
|
-
return;
|
|
11994
|
-
const rows = await sql`
|
|
11995
|
-
SELECT obj_description(c.oid, 'pg_class') AS definition
|
|
11996
|
-
FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid
|
|
11997
|
-
WHERE i.indrelid = 'objects'::regclass AND i.indisvalid
|
|
11998
|
-
AND c.relname LIKE 'sixb_query_%'
|
|
11999
|
-
AND obj_description(c.oid, 'pg_class') IS NOT NULL
|
|
12000
|
-
`;
|
|
12001
|
-
const matches = [];
|
|
12002
|
-
for (const row of rows) {
|
|
12003
|
-
let candidate;
|
|
12004
|
-
try {
|
|
12005
|
-
candidate = JSON.parse(row.definition);
|
|
12006
|
-
} catch {
|
|
12007
|
-
continue;
|
|
12008
|
-
}
|
|
12009
|
-
if (!candidate || typeof candidate !== "object")
|
|
12010
|
-
continue;
|
|
12011
|
-
const index = candidate;
|
|
12012
|
-
if (index.kind !== "sort" || index.projectId !== projectId || index.objectTypeId !== objectTypeId || index.where || !Array.isArray(index.fields) || index.fields.length !== fields.length)
|
|
12013
|
-
continue;
|
|
12014
|
-
if (index.equality !== undefined && (!Array.isArray(index.equality) || !index.equality.every((property) => typeof property === "string" && equality.has(property))))
|
|
12015
|
-
continue;
|
|
12016
|
-
if (!index.fields.every((field, i) => field && typeof field.propertyId === "string" && fields[i]?.kind === "property" && fields[i].propertyId === field.propertyId && (field.direction ?? "asc") === (fields[i].direction ?? "asc") && field.scalarKind === "decimal" === (fields[i].scalarKind === "decimal") && isTextScalarKind(field.scalarKind) === isTextScalarKind(fields[i].scalarKind)))
|
|
12017
|
-
continue;
|
|
12018
|
-
matches.push(index);
|
|
12019
|
-
}
|
|
12020
|
-
matches.sort((a, b) => (b.equality?.length ?? 0) - (a.equality?.length ?? 0));
|
|
12021
|
-
return matches[0] ? sortIndexBound(matches[0]) : undefined;
|
|
12453
|
+
AND selection.source_object_type_id = parent.object_type_id
|
|
12454
|
+
JOIN links AS edge
|
|
12455
|
+
ON edge.project_id = parent.project_id
|
|
12456
|
+
AND edge.source_type_id = parent.object_type_id
|
|
12457
|
+
AND edge.source_id = parent.primary_id
|
|
12458
|
+
AND edge.link_id = selection.link_id
|
|
12459
|
+
AND edge.target_type_id = selection.target_object_type_id
|
|
12460
|
+
JOIN objects AS target
|
|
12461
|
+
ON target.project_id = edge.project_id
|
|
12462
|
+
AND target.object_type_id = edge.target_type_id
|
|
12463
|
+
AND target.primary_id = edge.target_id
|
|
12464
|
+
),
|
|
12465
|
+
_sixb_scope_object_permissions(project_id, object_type_id, primary_id, property_id) AS (
|
|
12466
|
+
SELECT DISTINCT
|
|
12467
|
+
reachable.project_id,
|
|
12468
|
+
reachable.object_type_id,
|
|
12469
|
+
reachable.primary_id,
|
|
12470
|
+
selected_property.value
|
|
12471
|
+
FROM _sixb_scope_reachable AS reachable
|
|
12472
|
+
JOIN _sixb_scope_object_selections AS selection
|
|
12473
|
+
ON selection.node_id = reachable.node_id
|
|
12474
|
+
AND selection.object_type_id = reachable.object_type_id
|
|
12475
|
+
CROSS JOIN LATERAL jsonb_array_elements_text(selection.property_ids)
|
|
12476
|
+
AS selected_property(value)
|
|
12477
|
+
),
|
|
12478
|
+
_sixb_scope_object_ids(project_id, object_type_id, primary_id) AS (
|
|
12479
|
+
SELECT DISTINCT project_id, object_type_id, primary_id
|
|
12480
|
+
FROM _sixb_scope_reachable
|
|
12481
|
+
),
|
|
12482
|
+
_sixb_scope_object_grants AS (
|
|
12483
|
+
SELECT reachable.project_id, reachable.object_type_id, reachable.primary_id,
|
|
12484
|
+
array_agg(DISTINCT selected_property.value) FILTER (WHERE selected_property.value IS NOT NULL) AS property_ids
|
|
12485
|
+
FROM _sixb_scope_reachable AS reachable
|
|
12486
|
+
LEFT JOIN _sixb_scope_object_selections AS selection
|
|
12487
|
+
ON selection.node_id = reachable.node_id
|
|
12488
|
+
AND selection.object_type_id = reachable.object_type_id
|
|
12489
|
+
LEFT JOIN LATERAL jsonb_array_elements_text(selection.property_ids) AS selected_property(value) ON true
|
|
12490
|
+
GROUP BY reachable.project_id, reachable.object_type_id, reachable.primary_id
|
|
12491
|
+
),
|
|
12492
|
+
_sixb_scope_objects AS (
|
|
12493
|
+
SELECT stored.project_id, stored.object_type_id, stored.primary_id,
|
|
12494
|
+
COALESCE((
|
|
12495
|
+
SELECT jsonb_object_agg(property.key, property.value)
|
|
12496
|
+
FROM jsonb_each(stored.properties) AS property(key, value)
|
|
12497
|
+
WHERE property.key = ANY(grants.property_ids)
|
|
12498
|
+
), '{}'::jsonb) AS properties,
|
|
12499
|
+
stored.created_at, stored.updated_at, stored.version, stored.last_commit_id
|
|
12500
|
+
FROM _sixb_scope_object_grants AS grants
|
|
12501
|
+
JOIN objects AS stored
|
|
12502
|
+
ON stored.project_id = grants.project_id
|
|
12503
|
+
AND stored.object_type_id = grants.object_type_id
|
|
12504
|
+
AND stored.primary_id = grants.primary_id
|
|
12505
|
+
),
|
|
12506
|
+
_sixb_scope_reached_links(
|
|
12507
|
+
project_id,
|
|
12508
|
+
step_id,
|
|
12509
|
+
node_id,
|
|
12510
|
+
parent_node_id,
|
|
12511
|
+
source_type_id,
|
|
12512
|
+
source_id,
|
|
12513
|
+
link_id,
|
|
12514
|
+
target_type_id,
|
|
12515
|
+
target_id,
|
|
12516
|
+
property_ids
|
|
12517
|
+
) AS (
|
|
12518
|
+
SELECT
|
|
12519
|
+
edge.project_id,
|
|
12520
|
+
selection.step_id,
|
|
12521
|
+
selection.node_id,
|
|
12522
|
+
selection.parent_node_id,
|
|
12523
|
+
edge.source_type_id,
|
|
12524
|
+
edge.source_id,
|
|
12525
|
+
edge.link_id,
|
|
12526
|
+
edge.target_type_id,
|
|
12527
|
+
edge.target_id,
|
|
12528
|
+
selection.property_ids
|
|
12529
|
+
FROM _sixb_scope_reachable AS parent
|
|
12530
|
+
JOIN _sixb_scope_link_selections AS selection
|
|
12531
|
+
ON selection.parent_node_id = parent.node_id
|
|
12532
|
+
AND selection.source_object_type_id = parent.object_type_id
|
|
12533
|
+
JOIN links AS edge
|
|
12534
|
+
ON edge.project_id = parent.project_id
|
|
12535
|
+
AND edge.source_type_id = parent.object_type_id
|
|
12536
|
+
AND edge.source_id = parent.primary_id
|
|
12537
|
+
AND edge.link_id = selection.link_id
|
|
12538
|
+
AND edge.target_type_id = selection.target_object_type_id
|
|
12539
|
+
JOIN objects AS target
|
|
12540
|
+
ON target.project_id = edge.project_id
|
|
12541
|
+
AND target.object_type_id = edge.target_type_id
|
|
12542
|
+
AND target.primary_id = edge.target_id
|
|
12543
|
+
),
|
|
12544
|
+
_sixb_scope_link_permissions(
|
|
12545
|
+
project_id,
|
|
12546
|
+
source_type_id,
|
|
12547
|
+
source_id,
|
|
12548
|
+
link_id,
|
|
12549
|
+
target_type_id,
|
|
12550
|
+
target_id,
|
|
12551
|
+
property_id
|
|
12552
|
+
) AS (
|
|
12553
|
+
SELECT DISTINCT
|
|
12554
|
+
reached.project_id,
|
|
12555
|
+
reached.source_type_id,
|
|
12556
|
+
reached.source_id,
|
|
12557
|
+
reached.link_id,
|
|
12558
|
+
reached.target_type_id,
|
|
12559
|
+
reached.target_id,
|
|
12560
|
+
selected_property.value
|
|
12561
|
+
FROM _sixb_scope_reached_links AS reached
|
|
12562
|
+
CROSS JOIN LATERAL jsonb_array_elements_text(reached.property_ids)
|
|
12563
|
+
AS selected_property(value)
|
|
12564
|
+
),
|
|
12565
|
+
_sixb_scope_link_ids(
|
|
12566
|
+
project_id,
|
|
12567
|
+
source_type_id,
|
|
12568
|
+
source_id,
|
|
12569
|
+
link_id,
|
|
12570
|
+
target_type_id,
|
|
12571
|
+
target_id
|
|
12572
|
+
) AS (
|
|
12573
|
+
SELECT DISTINCT
|
|
12574
|
+
project_id,
|
|
12575
|
+
source_type_id,
|
|
12576
|
+
source_id,
|
|
12577
|
+
link_id,
|
|
12578
|
+
target_type_id,
|
|
12579
|
+
target_id
|
|
12580
|
+
FROM _sixb_scope_reached_links
|
|
12581
|
+
),
|
|
12582
|
+
_sixb_scope_links AS (
|
|
12583
|
+
SELECT
|
|
12584
|
+
stored.project_id,
|
|
12585
|
+
stored.source_type_id,
|
|
12586
|
+
stored.source_id,
|
|
12587
|
+
stored.link_id,
|
|
12588
|
+
stored.target_type_id,
|
|
12589
|
+
stored.target_id,
|
|
12590
|
+
CASE
|
|
12591
|
+
WHEN stored.properties IS NULL THEN NULL
|
|
12592
|
+
ELSE NULLIF(
|
|
12593
|
+
(
|
|
12594
|
+
SELECT jsonb_object_agg(property.key, property.value)
|
|
12595
|
+
FROM jsonb_each(stored.properties) AS property(key, value)
|
|
12596
|
+
JOIN _sixb_scope_link_permissions AS permission
|
|
12597
|
+
ON permission.project_id = stored.project_id
|
|
12598
|
+
AND permission.source_type_id = stored.source_type_id
|
|
12599
|
+
AND permission.source_id = stored.source_id
|
|
12600
|
+
AND permission.link_id = stored.link_id
|
|
12601
|
+
AND permission.target_type_id = stored.target_type_id
|
|
12602
|
+
AND permission.target_id = stored.target_id
|
|
12603
|
+
AND permission.property_id = property.key
|
|
12604
|
+
),
|
|
12605
|
+
'{}'::jsonb
|
|
12606
|
+
)
|
|
12607
|
+
END AS properties,
|
|
12608
|
+
stored.created_at,
|
|
12609
|
+
stored.updated_at,
|
|
12610
|
+
stored.last_commit_id
|
|
12611
|
+
FROM links AS stored
|
|
12612
|
+
JOIN _sixb_scope_link_ids AS visible
|
|
12613
|
+
ON visible.project_id = stored.project_id
|
|
12614
|
+
AND visible.source_type_id = stored.source_type_id
|
|
12615
|
+
AND visible.source_id = stored.source_id
|
|
12616
|
+
AND visible.link_id = stored.link_id
|
|
12617
|
+
AND visible.target_type_id = stored.target_type_id
|
|
12618
|
+
AND visible.target_id = stored.target_id
|
|
12619
|
+
)
|
|
12620
|
+
`,
|
|
12621
|
+
args: [scopeJson, projectId]
|
|
12622
|
+
};
|
|
12022
12623
|
}
|
|
12023
12624
|
|
|
12625
|
+
// src/objects/reader.ts
|
|
12626
|
+
import { ObjectQueryExecutionError as ObjectQueryExecutionError2 } from "@sixb/core";
|
|
12627
|
+
import { hasVectorProfile as hasVectorProfile2 } from "@sixb/core/internal/query";
|
|
12628
|
+
import { linkBatchKey, objectBatchKey } from "@sixb/core/storage";
|
|
12629
|
+
|
|
12024
12630
|
// src/objects/rows.ts
|
|
12025
12631
|
function rowToObject(row) {
|
|
12026
12632
|
return {
|
|
@@ -12106,7 +12712,7 @@ function linkIdentity(link) {
|
|
|
12106
12712
|
}
|
|
12107
12713
|
|
|
12108
12714
|
// src/objects/text-count-index.ts
|
|
12109
|
-
import { createHash as
|
|
12715
|
+
import { createHash as createHash5 } from "crypto";
|
|
12110
12716
|
function describe(input) {
|
|
12111
12717
|
const definition = { ...input, filters: [...new Set(input.filters ?? [])].sort() };
|
|
12112
12718
|
if (definition.filters.length > 4)
|
|
@@ -12117,7 +12723,7 @@ function describe(input) {
|
|
|
12117
12723
|
input.propertyId,
|
|
12118
12724
|
...definition.filters
|
|
12119
12725
|
])
|
|
12120
|
-
|
|
12726
|
+
literal(value);
|
|
12121
12727
|
const key = hash(JSON.stringify([
|
|
12122
12728
|
definition.projectId,
|
|
12123
12729
|
definition.objectTypeId,
|
|
@@ -12129,10 +12735,10 @@ function describe(input) {
|
|
|
12129
12735
|
property,
|
|
12130
12736
|
`sixb_qfilter_${hash(JSON.stringify([input.projectId, input.objectTypeId, property]))}`
|
|
12131
12737
|
]));
|
|
12132
|
-
const scope = `project_id = ${
|
|
12738
|
+
const scope = `project_id = ${literal(input.projectId)} AND object_type_id = ${literal(input.objectTypeId)}`;
|
|
12133
12739
|
const compact = [
|
|
12134
|
-
`octet_length(${
|
|
12135
|
-
...[...filters.values()].map((column) => `(${
|
|
12740
|
+
`octet_length(${identifier(textColumn)}) <= 1024`,
|
|
12741
|
+
...[...filters.values()].map((column) => `(${identifier(column)} IS NULL OR octet_length(${identifier(column)}) <= 128)`)
|
|
12136
12742
|
].join(" AND ");
|
|
12137
12743
|
return { definition, name: `sixb_text_count_${key}`, textColumn, filters, scope, compact };
|
|
12138
12744
|
}
|
|
@@ -12148,7 +12754,7 @@ async function preparePgObjectTextCounts(sql, schema, inputs) {
|
|
|
12148
12754
|
`;
|
|
12149
12755
|
if (!extension?.ready)
|
|
12150
12756
|
throw new Error("[SixbPg] Text counts require CREATE EXTENSION pg_trgm WITH SCHEMA public");
|
|
12151
|
-
const table = `${
|
|
12757
|
+
const table = `${identifier(schema)}.objects`;
|
|
12152
12758
|
const names = [];
|
|
12153
12759
|
await withReservedPgConnection(sql, async (connection) => {
|
|
12154
12760
|
let connected = true;
|
|
@@ -12161,29 +12767,36 @@ async function preparePgObjectTextCounts(sql, schema, inputs) {
|
|
|
12161
12767
|
const columns = new Set(existing.map((row) => row.attname));
|
|
12162
12768
|
const additions = new Map;
|
|
12163
12769
|
for (const item of definitions) {
|
|
12164
|
-
additions.set(item.textColumn, `CASE WHEN ${item.scope} THEN lower(coalesce(properties ->> ${
|
|
12770
|
+
additions.set(item.textColumn, `CASE WHEN ${item.scope} THEN lower(coalesce(properties ->> ${literal(item.definition.propertyId)}, '')) END`);
|
|
12165
12771
|
for (const [property, column] of item.filters)
|
|
12166
|
-
additions.set(column, `CASE WHEN ${item.scope} AND jsonb_typeof(properties -> ${
|
|
12772
|
+
additions.set(column, `CASE WHEN ${item.scope} AND jsonb_typeof(properties -> ${literal(property)}) = 'string' THEN properties ->> ${literal(property)} END`);
|
|
12167
12773
|
}
|
|
12168
12774
|
const missing = [...additions].filter(([column]) => !columns.has(column));
|
|
12169
12775
|
if (missing.length > 0) {
|
|
12170
|
-
await connection.unsafe(`ALTER TABLE ${table} ${missing.map(([column, expression]) => `ADD COLUMN ${
|
|
12776
|
+
await connection.unsafe(`ALTER TABLE ${table} ${missing.map(([column, expression]) => `ADD COLUMN ${identifier(column)} text GENERATED ALWAYS AS (${expression}) STORED`).join(", ")}`);
|
|
12777
|
+
}
|
|
12778
|
+
const objectTypeIds = new Set(definitions.map((item) => item.definition.objectTypeId));
|
|
12779
|
+
for (const objectTypeId of objectTypeIds) {
|
|
12780
|
+
await ensureObjectPartition(connection, schema, objectTypeId);
|
|
12171
12781
|
}
|
|
12172
12782
|
for (const item of definitions) {
|
|
12173
12783
|
const statements = textCountIndexStatements(item);
|
|
12784
|
+
const partition = `${identifier(schema)}.${identifier(objectPartitionName(item.definition.objectTypeId))}`;
|
|
12174
12785
|
for (const [name, expression] of statements) {
|
|
12175
|
-
await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${
|
|
12786
|
+
await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${partition} ${expression}`);
|
|
12176
12787
|
const [valid] = await connection`
|
|
12177
12788
|
SELECT indisvalid FROM pg_index
|
|
12178
|
-
WHERE indexrelid = ${`${
|
|
12789
|
+
WHERE indexrelid = ${`${identifier(schema)}.${identifier(name)}`}::regclass
|
|
12179
12790
|
`;
|
|
12180
12791
|
if (!valid?.indisvalid)
|
|
12181
12792
|
throw new Error(`[SixbPg] Text count index ${schema}.${name} is invalid. Drop it concurrently and retry preparation.`);
|
|
12182
12793
|
}
|
|
12183
|
-
await connection.unsafe(`COMMENT ON INDEX ${
|
|
12794
|
+
await connection.unsafe(`COMMENT ON INDEX ${identifier(schema)}.${identifier(item.name)} IS ${literal(JSON.stringify(item.definition))}`);
|
|
12184
12795
|
names.push(item.name);
|
|
12185
12796
|
}
|
|
12186
|
-
|
|
12797
|
+
for (const objectTypeId of objectTypeIds) {
|
|
12798
|
+
await connection.unsafe(`ANALYZE ${identifier(schema)}.${identifier(objectPartitionName(objectTypeId))}`);
|
|
12799
|
+
}
|
|
12187
12800
|
} catch (error) {
|
|
12188
12801
|
connected = !isConnectionLost(error);
|
|
12189
12802
|
throw error;
|
|
@@ -12199,16 +12812,16 @@ function textCountIndexStatements(item) {
|
|
|
12199
12812
|
const statements = [
|
|
12200
12813
|
[
|
|
12201
12814
|
item.name,
|
|
12202
|
-
`(${[...item.filters.values(), item.textColumn].map(
|
|
12815
|
+
`(${[...item.filters.values(), item.textColumn].map(identifier).join(", ")}) WHERE ${predicate}`
|
|
12203
12816
|
],
|
|
12204
12817
|
[
|
|
12205
12818
|
`${item.name}_gin`,
|
|
12206
|
-
`USING gin (${
|
|
12819
|
+
`USING gin (${identifier(item.textColumn)} public.gin_trgm_ops) WHERE ${predicate}`
|
|
12207
12820
|
],
|
|
12208
12821
|
[`${item.name}_overflow`, `(primary_id) WHERE ${item.scope} AND NOT (${item.compact})`]
|
|
12209
12822
|
];
|
|
12210
12823
|
if (item.filters.size > 0) {
|
|
12211
|
-
const columns = [...item.filters.values()].map(
|
|
12824
|
+
const columns = [...item.filters.values()].map(identifier);
|
|
12212
12825
|
const compact = columns.map((column) => `(${column} IS NOT NULL AND octet_length(${column}) <= 128)`).join(" AND ");
|
|
12213
12826
|
statements.push([`${item.name}_facets`, `(${columns.join(", ")}) WHERE ${item.scope} AND (${compact})`], [`${item.name}_facet_overflow`, `(primary_id) WHERE ${item.scope} AND NOT (${compact})`], [
|
|
12214
12827
|
`${item.name}_text_facets_overflow`,
|
|
@@ -12230,7 +12843,7 @@ async function compilePgIndexedTextAggregate(sql, projectId, query, facets) {
|
|
|
12230
12843
|
const rows = await sql`
|
|
12231
12844
|
SELECT c.relname AS name, obj_description(c.oid,'pg_class') AS definition
|
|
12232
12845
|
FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid
|
|
12233
|
-
WHERE i.indrelid=
|
|
12846
|
+
WHERE i.indrelid=to_regclass(${objectPartitionName(shape.objectTypeId)}) AND i.indisvalid
|
|
12234
12847
|
AND c.relname LIKE 'sixb_text_count_%' AND obj_description(c.oid,'pg_class') IS NOT NULL`;
|
|
12235
12848
|
for (const row of rows) {
|
|
12236
12849
|
const definition = parseDefinition(row.definition);
|
|
@@ -12275,8 +12888,8 @@ async function compilePgIndexedTextAggregate(sql, projectId, query, facets) {
|
|
|
12275
12888
|
broadWhere.push("false");
|
|
12276
12889
|
}
|
|
12277
12890
|
for (const term of terms) {
|
|
12278
|
-
where.push(`${
|
|
12279
|
-
broadWhere.push(`position(${bind(term)}::text in ${
|
|
12891
|
+
where.push(`${identifier(item.textColumn)} LIKE ${bind(`%${term.replace(/[\\%_]/g, "\\$&")}%`)}::text`);
|
|
12892
|
+
broadWhere.push(`position(${bind(term)}::text in ${identifier(item.textColumn)}) > 0`);
|
|
12280
12893
|
}
|
|
12281
12894
|
}
|
|
12282
12895
|
const input = { projectId, query, item, where, broadWhere, args };
|
|
@@ -12287,12 +12900,12 @@ async function compilePgIndexedTextAggregate(sql, projectId, query, facets) {
|
|
|
12287
12900
|
return;
|
|
12288
12901
|
}
|
|
12289
12902
|
function compileIndexedFacets({ projectId, query, item, where, broadWhere, args }, facets, hasText, bind) {
|
|
12290
|
-
const allString = [...item.filters.values()].map((column) => `${
|
|
12291
|
-
const filterCompact = [...item.filters.values()].map((column) => `(${
|
|
12903
|
+
const allString = [...item.filters.values()].map((column) => `${identifier(column)} IS NOT NULL`).join(" AND ") || "true";
|
|
12904
|
+
const filterCompact = [...item.filters.values()].map((column) => `(${identifier(column)} IS NOT NULL AND octet_length(${identifier(column)}) <= 128)`).join(" AND ") || "true";
|
|
12292
12905
|
const compact = hasText ? `(${item.compact}) AND (${allString})` : filterCompact;
|
|
12293
12906
|
const nativeWhere = hasText ? [...where, allString] : [where[0], where[1], compact, ...where.slice(3)];
|
|
12294
12907
|
const nativeBroad = [...broadWhere, allString];
|
|
12295
|
-
const fields = facets.map((facet, i) => `'string'::text AS f${i}_type, ${
|
|
12908
|
+
const fields = facets.map((facet, i) => `'string'::text AS f${i}_type, ${identifier(item.filters.get(facet.propertyId))} AS f${i}_value`).join(", ");
|
|
12296
12909
|
const overflow = compilePgObjectQuery(projectId, query, {
|
|
12297
12910
|
includeTotal: false,
|
|
12298
12911
|
source: {
|
|
@@ -12370,7 +12983,7 @@ function stringPredicate(predicate, columns, bind) {
|
|
|
12370
12983
|
const values = predicate.op === "in" ? predicate.values : [predicate.value];
|
|
12371
12984
|
if (!values.every((value) => typeof value === "string"))
|
|
12372
12985
|
return;
|
|
12373
|
-
return values.length ? `${
|
|
12986
|
+
return values.length ? `${identifier(column)} IN (${values.map((value) => `${bind(value)}::text`).join(", ")})` : "false";
|
|
12374
12987
|
}
|
|
12375
12988
|
function parseDefinition(text) {
|
|
12376
12989
|
try {
|
|
@@ -12388,15 +13001,7 @@ function parseDefinition(text) {
|
|
|
12388
13001
|
}
|
|
12389
13002
|
}
|
|
12390
13003
|
function hash(value) {
|
|
12391
|
-
return
|
|
12392
|
-
}
|
|
12393
|
-
function identifier2(value) {
|
|
12394
|
-
return `"${value.replace(/"/g, '""')}"`;
|
|
12395
|
-
}
|
|
12396
|
-
function literal2(value) {
|
|
12397
|
-
if (value.includes("\x00"))
|
|
12398
|
-
throw new Error("[SixbPg] Text count identifiers cannot contain NUL");
|
|
12399
|
-
return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`;
|
|
13004
|
+
return createHash5("sha256").update(value).digest("hex").slice(0, 24);
|
|
12400
13005
|
}
|
|
12401
13006
|
|
|
12402
13007
|
// src/objects/reader.ts
|
|
@@ -12404,10 +13009,12 @@ class PgObjectReader {
|
|
|
12404
13009
|
sql;
|
|
12405
13010
|
source;
|
|
12406
13011
|
aggregateAdmission;
|
|
12407
|
-
|
|
13012
|
+
sortIndexes;
|
|
13013
|
+
constructor(sql, source, aggregateAdmission, sortIndexes) {
|
|
12408
13014
|
this.sql = sql;
|
|
12409
13015
|
this.source = source;
|
|
12410
13016
|
this.aggregateAdmission = aggregateAdmission;
|
|
13017
|
+
this.sortIndexes = sortIndexes;
|
|
12411
13018
|
}
|
|
12412
13019
|
async assertVectorSearch(params) {
|
|
12413
13020
|
if (!hasVectorProfile2(params.query))
|
|
@@ -12447,7 +13054,7 @@ class PgObjectReader {
|
|
|
12447
13054
|
async compilePageQuery(params) {
|
|
12448
13055
|
const sql = this.sql;
|
|
12449
13056
|
const source = this.source;
|
|
12450
|
-
const sortIndexBound = source === DEFAULT_OBJECT_QUERY_SOURCE ? await findPgSortIndexBound(sql, params.projectId, params.query) : undefined;
|
|
13057
|
+
const sortIndexBound = source === DEFAULT_OBJECT_QUERY_SOURCE && this.sortIndexes ? await findPgSortIndexBound(sql, this.sortIndexes, params.projectId, params.query) : undefined;
|
|
12451
13058
|
let compiled = compilePgObjectQuery(params.projectId, params.query, {
|
|
12452
13059
|
sortIndexBound,
|
|
12453
13060
|
includeTotal: params.includeTotal,
|
|
@@ -13089,12 +13696,14 @@ var PG_OBJECT_QUERY_CAPABILITIES = {
|
|
|
13089
13696
|
|
|
13090
13697
|
class PgObjectStorage {
|
|
13091
13698
|
sql;
|
|
13699
|
+
sortIndexes;
|
|
13092
13700
|
aggregateAdmission;
|
|
13093
13701
|
reader;
|
|
13094
|
-
constructor(sql, maxConcurrentAggregates = 1) {
|
|
13702
|
+
constructor(sql, sortIndexes = new PgSortIndexCatalog, maxConcurrentAggregates = 1) {
|
|
13095
13703
|
this.sql = sql;
|
|
13704
|
+
this.sortIndexes = sortIndexes;
|
|
13096
13705
|
this.aggregateAdmission = canStartPgTransaction(sql) ? new PgAggregateAdmission(maxConcurrentAggregates) : undefined;
|
|
13097
|
-
this.reader = new PgObjectReader(sql, DEFAULT_OBJECT_QUERY_SOURCE, this.aggregateAdmission);
|
|
13706
|
+
this.reader = new PgObjectReader(sql, DEFAULT_OBJECT_QUERY_SOURCE, this.aggregateAdmission, sortIndexes);
|
|
13098
13707
|
}
|
|
13099
13708
|
queryCapabilities() {
|
|
13100
13709
|
return PG_OBJECT_QUERY_CAPABILITIES;
|
|
@@ -13110,7 +13719,7 @@ class PgObjectStorage {
|
|
|
13110
13719
|
}
|
|
13111
13720
|
async queryObjects(params) {
|
|
13112
13721
|
if (hasVectorProfile4(params.query))
|
|
13113
|
-
return runPgRepeatableReadTransaction(this.sql, (tx) => new PgObjectReader(tx, DEFAULT_OBJECT_QUERY_SOURCE).queryObjects(params));
|
|
13722
|
+
return runPgRepeatableReadTransaction(this.sql, (tx) => new PgObjectReader(tx, DEFAULT_OBJECT_QUERY_SOURCE, undefined, this.sortIndexes).queryObjects(params));
|
|
13114
13723
|
return this.reader.queryObjects(params);
|
|
13115
13724
|
}
|
|
13116
13725
|
async countObjects(params) {
|
|
@@ -13207,7 +13816,7 @@ function assertReconciliationPageLimit(limit) {
|
|
|
13207
13816
|
}
|
|
13208
13817
|
}
|
|
13209
13818
|
// src/objects/prepare-queries.ts
|
|
13210
|
-
import { createHash as
|
|
13819
|
+
import { createHash as createHash6 } from "crypto";
|
|
13211
13820
|
async function preparePgObjectQueries(sql, schema, plan) {
|
|
13212
13821
|
const fingerprint = preparationFingerprint(plan);
|
|
13213
13822
|
const current = await readCompletedPreparation(sql, fingerprint);
|
|
@@ -13232,17 +13841,18 @@ async function preparePgObjectQueries(sql, schema, plan) {
|
|
|
13232
13841
|
return result;
|
|
13233
13842
|
}, unlock);
|
|
13234
13843
|
await unlock();
|
|
13235
|
-
return result;
|
|
13844
|
+
return result.status === "current" ? result : { ...result, warnings: [...result.warnings, ...await linkWarnings(connection, plan)] };
|
|
13236
13845
|
});
|
|
13237
13846
|
}
|
|
13238
13847
|
function preparationFingerprint(plan) {
|
|
13239
13848
|
const objectTypes = plan.objectTypes.map((type) => ({
|
|
13240
13849
|
objectTypeId: type.objectTypeId,
|
|
13241
13850
|
properties: [...type.properties].sort((a, b) => a.propertyId.localeCompare(b.propertyId)),
|
|
13242
|
-
indexes: type.indexes.map(normalizeIndex).sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)))
|
|
13851
|
+
indexes: type.indexes.map(normalizeIndex).sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
|
|
13852
|
+
links: [...type.links].sort((a, b) => a.linkId.localeCompare(b.linkId))
|
|
13243
13853
|
})).sort((a, b) => a.objectTypeId.localeCompare(b.objectTypeId));
|
|
13244
|
-
const definition = { version:
|
|
13245
|
-
return
|
|
13854
|
+
const definition = { version: 3, projectId: plan.projectId, objectTypes };
|
|
13855
|
+
return createHash6("sha256").update(JSON.stringify(definition)).digest("hex");
|
|
13246
13856
|
}
|
|
13247
13857
|
function normalizeIndex(index) {
|
|
13248
13858
|
if (index.kind === "sort") {
|
|
@@ -13275,6 +13885,8 @@ async function prepareStructures(sql, schema, plan) {
|
|
|
13275
13885
|
if (indexes.some((index) => index.kind === "text") || textCounts.length > 0) {
|
|
13276
13886
|
await ensureTrigrams(sql);
|
|
13277
13887
|
}
|
|
13888
|
+
for (const type of plan.objectTypes)
|
|
13889
|
+
await ensureObjectPartition(sql, schema, type.objectTypeId);
|
|
13278
13890
|
const names = await ensurePgObjectQueryIndexes(sql, schema, indexes);
|
|
13279
13891
|
const texts = await preparePgObjectTextCounts(sql, schema, textCounts);
|
|
13280
13892
|
return {
|
|
@@ -13284,6 +13896,27 @@ async function prepareStructures(sql, schema, plan) {
|
|
|
13284
13896
|
warnings
|
|
13285
13897
|
};
|
|
13286
13898
|
}
|
|
13899
|
+
async function linkWarnings(sql, plan) {
|
|
13900
|
+
const declared = new Map(plan.objectTypes.flatMap((type) => type.links.map((link) => [`${type.objectTypeId}\x00${link.linkId}`, link])));
|
|
13901
|
+
try {
|
|
13902
|
+
const edges = await sql`
|
|
13903
|
+
SELECT source_type_id, link_id, target_type_id, COUNT(*) AS edges
|
|
13904
|
+
FROM links WHERE project_id = ${plan.projectId}
|
|
13905
|
+
GROUP BY source_type_id, link_id, target_type_id`;
|
|
13906
|
+
return edges.flatMap((edge) => {
|
|
13907
|
+
const name = `${edge.source_type_id}.${edge.link_id}`;
|
|
13908
|
+
const link = declared.get(`${edge.source_type_id}\x00${edge.link_id}`);
|
|
13909
|
+
if (!link) {
|
|
13910
|
+
return `${name}: ${edge.edges} stored edges to ${edge.target_type_id} belong to a link the ontology no longer declares. Queries may not return them.`;
|
|
13911
|
+
}
|
|
13912
|
+
return !link.targetObjectTypeIds || link.targetObjectTypeIds.includes(edge.target_type_id) ? [] : `${name}: ${edge.edges} stored edges point to ${edge.target_type_id}, which this link no longer allows. Queries may not return them.`;
|
|
13913
|
+
});
|
|
13914
|
+
} catch (error) {
|
|
13915
|
+
if (isConnectionLost(error))
|
|
13916
|
+
throw error;
|
|
13917
|
+
return [`Could not check stored links against the ontology: ${String(error)}`];
|
|
13918
|
+
}
|
|
13919
|
+
}
|
|
13287
13920
|
function compilePreparationPlan(plan) {
|
|
13288
13921
|
const indexes = [];
|
|
13289
13922
|
const textCounts = [];
|
|
@@ -13543,12 +14176,16 @@ class PgOntologyCommitStorage {
|
|
|
13543
14176
|
assertNonnegativeInteger(offset, "Ontology commit list offset");
|
|
13544
14177
|
if (limit !== undefined)
|
|
13545
14178
|
assertNonnegativeInteger(limit, "Ontology commit list limit");
|
|
14179
|
+
if (input.committedAt !== undefined) {
|
|
14180
|
+
assertTimestamp(input.committedAt, "Ontology commit list committedAt");
|
|
14181
|
+
}
|
|
13546
14182
|
const runFilter = input.run?.kind === "action" ? sql`AND origin_kind = 'action' AND origin_run_id = ${input.run.id}` : input.run?.kind === "projection" ? sql`AND origin_run_id = ${input.run.id}
|
|
13547
14183
|
AND origin_kind IN ('projection', 'telemetry')` : sql``;
|
|
14184
|
+
const committedAtFilter = input.committedAt === undefined ? sql`` : sql`AND committed_at = ${input.committedAt}::timestamptz`;
|
|
13548
14185
|
const [{ total }] = await sql`
|
|
13549
14186
|
SELECT COUNT(*) AS total
|
|
13550
14187
|
FROM ontology_commits
|
|
13551
|
-
WHERE project_id = ${input.projectId} ${runFilter}
|
|
14188
|
+
WHERE project_id = ${input.projectId} ${runFilter} ${committedAtFilter}
|
|
13552
14189
|
`;
|
|
13553
14190
|
const direction = input.order === "desc" ? sql`DESC` : sql`ASC`;
|
|
13554
14191
|
const order = input.run?.kind === "projection" ? sql`CASE WHEN origin_kind = 'telemetry' THEN origin_batch_ordinal ELSE -1 END
|
|
@@ -13556,7 +14193,7 @@ class PgOntologyCommitStorage {
|
|
|
13556
14193
|
const limitFragment = limit === undefined ? sql`` : sql`LIMIT ${limit}`;
|
|
13557
14194
|
const rows = await sql`
|
|
13558
14195
|
SELECT * FROM ontology_commits
|
|
13559
|
-
WHERE project_id = ${input.projectId} ${runFilter}
|
|
14196
|
+
WHERE project_id = ${input.projectId} ${runFilter} ${committedAtFilter}
|
|
13560
14197
|
ORDER BY ${order}
|
|
13561
14198
|
${limitFragment} OFFSET ${offset}
|
|
13562
14199
|
`;
|
|
@@ -13592,6 +14229,7 @@ import {
|
|
|
13592
14229
|
beginMaterializationVectorChanges,
|
|
13593
14230
|
cardinalityViolation,
|
|
13594
14231
|
invalidCorrelation as invalidCorrelation2,
|
|
14232
|
+
prepareMaterializationStateRead,
|
|
13595
14233
|
sameNonnegativeCounts as sameCounts,
|
|
13596
14234
|
uniqueSorted
|
|
13597
14235
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
@@ -15296,7 +15934,7 @@ async function sourceRow(sql, projectId, sourceId, materializationId, lock = fal
|
|
|
15296
15934
|
class PgMaterializationSessionState extends ProviderMaterializationSessionState {
|
|
15297
15935
|
plan;
|
|
15298
15936
|
stagedWorkCount = 0;
|
|
15299
|
-
changedObjects =
|
|
15937
|
+
changedObjects = new Map;
|
|
15300
15938
|
changedLinks = 0;
|
|
15301
15939
|
changedUniqueValues = 0;
|
|
15302
15940
|
uniqueWork = { releases: false, claims: false };
|
|
@@ -15344,6 +15982,9 @@ class PgMaterializationSessions {
|
|
|
15344
15982
|
return;
|
|
15345
15983
|
if (!session.plan)
|
|
15346
15984
|
await this.deleteWork(session.id);
|
|
15985
|
+
this.discard(session);
|
|
15986
|
+
}
|
|
15987
|
+
discard(session) {
|
|
15347
15988
|
session.active = false;
|
|
15348
15989
|
this.live.delete(session);
|
|
15349
15990
|
this.context?.materializations.complete(session.providerToken);
|
|
@@ -15705,6 +16346,18 @@ class PgMaterializationWriter {
|
|
|
15705
16346
|
await this.writeTouches(target);
|
|
15706
16347
|
return { objectWrites, linkWrites, uniqueValueWrites, eventCount };
|
|
15707
16348
|
}
|
|
16349
|
+
async stagedObjectWritesByType(target) {
|
|
16350
|
+
const rows = await this.sql`
|
|
16351
|
+
SELECT object_type_id, COUNT(*) AS writes FROM (
|
|
16352
|
+
SELECT value->'row'->'ref'->>'objectTypeId' AS object_type_id
|
|
16353
|
+
FROM (${this.staged(target, "object-upsert")}) AS staged
|
|
16354
|
+
UNION ALL
|
|
16355
|
+
SELECT value->'ref'->>'objectTypeId'
|
|
16356
|
+
FROM (${this.staged(target, "object-delete")}) AS staged
|
|
16357
|
+
) AS writes
|
|
16358
|
+
GROUP BY object_type_id`;
|
|
16359
|
+
return new Map(rows.map((row) => [row.object_type_id, Number(row.writes)]));
|
|
16360
|
+
}
|
|
15708
16361
|
async stagedUniqueWork(target) {
|
|
15709
16362
|
const probe = (kind) => this.sql`
|
|
15710
16363
|
EXISTS (
|
|
@@ -16286,7 +16939,7 @@ class PgOntologyMaterializationStorage {
|
|
|
16286
16939
|
}
|
|
16287
16940
|
return session.publicSession();
|
|
16288
16941
|
} catch (error) {
|
|
16289
|
-
|
|
16942
|
+
this.sessions.discard(session);
|
|
16290
16943
|
throw error;
|
|
16291
16944
|
}
|
|
16292
16945
|
}
|
|
@@ -16340,6 +16993,17 @@ class PgOntologyMaterializationStorage {
|
|
|
16340
16993
|
}
|
|
16341
16994
|
}
|
|
16342
16995
|
}
|
|
16996
|
+
async readState(input) {
|
|
16997
|
+
const request = prepareMaterializationStateRead(input);
|
|
16998
|
+
return runPgRepeatableReadTransaction(this.sql, async (tx) => {
|
|
16999
|
+
const reader = new PgMaterializationStateReader(tx, request.projectId);
|
|
17000
|
+
return {
|
|
17001
|
+
objects: await reader.objectStates(request.objects),
|
|
17002
|
+
links: await reader.linkStates(request.links),
|
|
17003
|
+
linkScopes: await reader.linkSlotStates(request.linkScopes)
|
|
17004
|
+
};
|
|
17005
|
+
});
|
|
17006
|
+
}
|
|
16343
17007
|
async stageWork(input) {
|
|
16344
17008
|
await this.sessions.stage(input);
|
|
16345
17009
|
}
|
|
@@ -16366,7 +17030,15 @@ class PgOntologyMaterializationStorage {
|
|
|
16366
17030
|
committedAt: commit.committedAt,
|
|
16367
17031
|
...session.plan ? {} : { uniqueWork: session.uniqueWork }
|
|
16368
17032
|
});
|
|
16369
|
-
|
|
17033
|
+
if (applied.objectWrites >= STATISTICS_MINIMUM_CHANGES) {
|
|
17034
|
+
session.changedObjects = await this.writer.stagedObjectWritesByType({
|
|
17035
|
+
workTable: session.workTable,
|
|
17036
|
+
workId: session.workId,
|
|
17037
|
+
projectId: commit.projectId,
|
|
17038
|
+
commitId: commit.id,
|
|
17039
|
+
committedAt: commit.committedAt
|
|
17040
|
+
});
|
|
17041
|
+
}
|
|
16370
17042
|
session.changedLinks = applied.linkWrites;
|
|
16371
17043
|
session.changedUniqueValues = applied.uniqueValueWrites;
|
|
16372
17044
|
session.appliedEventCount = applied.eventCount;
|
|
@@ -16379,7 +17051,7 @@ class PgOntologyMaterializationStorage {
|
|
|
16379
17051
|
for (const activation of input.finalization.sourceActivations) {
|
|
16380
17052
|
await this.activateSource(session, activation);
|
|
16381
17053
|
}
|
|
16382
|
-
await
|
|
17054
|
+
await refreshObjectStatistics(this.sql, session.changedObjects);
|
|
16383
17055
|
await refreshMaterializedStatistics(this.sql, "links", session.changedLinks);
|
|
16384
17056
|
await refreshMaterializedStatistics(this.sql, "object_unique_values", session.changedUniqueValues);
|
|
16385
17057
|
const record = await this.insertCommit(session.header, input);
|
|
@@ -16709,14 +17381,36 @@ function materializationLockKeys(header) {
|
|
|
16709
17381
|
function databaseCount(value) {
|
|
16710
17382
|
return Number(value ?? 0);
|
|
16711
17383
|
}
|
|
17384
|
+
var STATISTICS_MINIMUM_CHANGES = 1000;
|
|
16712
17385
|
async function refreshMaterializedStatistics(sql, table, changed) {
|
|
16713
|
-
if (changed <
|
|
17386
|
+
if (changed < STATISTICS_MINIMUM_CHANGES)
|
|
16714
17387
|
return;
|
|
16715
17388
|
const [state] = await sql`
|
|
16716
17389
|
SELECT reltuples AS rows FROM pg_class WHERE oid=${table}::regclass`;
|
|
16717
|
-
if (!state || state.rows <= 0 || changed >= Math.max(1e4, state.rows * 0.1))
|
|
17390
|
+
if (!state || state.rows <= 0 || changed >= state.rows || changed >= Math.max(1e4, state.rows * 0.1))
|
|
16718
17391
|
await sql.unsafe(`ANALYZE ${table}`);
|
|
16719
17392
|
}
|
|
17393
|
+
async function refreshObjectStatistics(sql, changes) {
|
|
17394
|
+
const total = [...changes.values()].reduce((sum, changed) => sum + changed, 0);
|
|
17395
|
+
if (total < STATISTICS_MINIMUM_CHANGES)
|
|
17396
|
+
return;
|
|
17397
|
+
const [parent] = await sql`
|
|
17398
|
+
SELECT reltuples AS rows FROM pg_class WHERE oid = 'objects'::regclass`;
|
|
17399
|
+
if (!parent || parent.rows <= 0 || total >= Math.max(1e4, parent.rows * 0.1)) {
|
|
17400
|
+
await sql.unsafe("ANALYZE objects");
|
|
17401
|
+
return;
|
|
17402
|
+
}
|
|
17403
|
+
for (const [objectTypeId, changed] of changes) {
|
|
17404
|
+
if (changed < STATISTICS_MINIMUM_CHANGES)
|
|
17405
|
+
continue;
|
|
17406
|
+
await refreshMaterializedStatistics(sql, await objectStatisticsTable(sql, objectTypeId), changed);
|
|
17407
|
+
}
|
|
17408
|
+
}
|
|
17409
|
+
async function objectStatisticsTable(sql, objectTypeId) {
|
|
17410
|
+
const partition = objectPartitionName(objectTypeId);
|
|
17411
|
+
const [row] = await sql`SELECT to_regclass(${partition})::text AS found`;
|
|
17412
|
+
return row?.found ? partition : "objects_default";
|
|
17413
|
+
}
|
|
16720
17414
|
|
|
16721
17415
|
// src/ontology-storage/outbox.ts
|
|
16722
17416
|
import { serializeSixbFailure as serializeSixbFailure2 } from "@sixb/core/internal/errors";
|
|
@@ -18587,41 +19281,88 @@ import {
|
|
|
18587
19281
|
AiCostStorageError
|
|
18588
19282
|
} from "@sixb/core/storage";
|
|
18589
19283
|
|
|
19284
|
+
// src/ai-accounting-filters.ts
|
|
19285
|
+
function pgAccountingFilterValues(query) {
|
|
19286
|
+
return {
|
|
19287
|
+
kind: query.modelKind ?? null,
|
|
19288
|
+
source: query.source?.kind ?? null,
|
|
19289
|
+
workflow: query.source?.kind === "workflow" ? query.source.workflowId ?? null : null,
|
|
19290
|
+
mode: query.requester === undefined ? null : query.requester === "none" ? "none" : query.requester.type,
|
|
19291
|
+
id: query.requester === undefined || query.requester === "none" ? null : query.requester.id
|
|
19292
|
+
};
|
|
19293
|
+
}
|
|
19294
|
+
function pgAccountingFilterSql(start) {
|
|
19295
|
+
const [kind, source, workflow, mode, id] = [0, 1, 2, 3, 4].map((offset) => `$${start + offset}`);
|
|
19296
|
+
const sourceWorkflow = `COALESCE(workflow_node.workflow_id,
|
|
19297
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END)`;
|
|
19298
|
+
return `(${kind}::text IS NULL OR usage.model_kind = ${kind})
|
|
19299
|
+
AND (${source}::text IS NULL OR (CASE
|
|
19300
|
+
WHEN direct_agent.id IS NOT NULL THEN 'agent'
|
|
19301
|
+
WHEN ${sourceWorkflow} IS NOT NULL THEN 'workflow'
|
|
19302
|
+
WHEN execution.executor_kind = 'action' THEN 'action'
|
|
19303
|
+
ELSE 'other'
|
|
19304
|
+
END) = ${source})
|
|
19305
|
+
AND (${workflow}::text IS NULL OR ${sourceWorkflow} = ${workflow})
|
|
19306
|
+
AND (
|
|
19307
|
+
${mode}::text IS NULL
|
|
19308
|
+
OR (${mode} = 'none' AND execution.requested_by_user_id IS NULL
|
|
19309
|
+
AND execution.requested_by_service_account_id IS NULL)
|
|
19310
|
+
OR (${mode} = 'user' AND execution.requested_by_user_id = ${id})
|
|
19311
|
+
OR (${mode} = 'serviceAccount' AND execution.requested_by_service_account_id = ${id})
|
|
19312
|
+
)`;
|
|
19313
|
+
}
|
|
19314
|
+
|
|
18590
19315
|
// src/ai-cost-groups.ts
|
|
18591
19316
|
import {
|
|
18592
19317
|
aiModelCallGroupFragmentFromRow,
|
|
18593
19318
|
buildAiModelCallGroupsFromFragments,
|
|
18594
19319
|
normalizeAiModelCallAccountingQuery,
|
|
19320
|
+
normalizeAiModelCallGroupSort,
|
|
18595
19321
|
TOKENLESS_MODEL_KINDS
|
|
18596
19322
|
} from "@sixb/core/internal/ai-cost-storage-provider";
|
|
18597
19323
|
async function listAiModelCallGroups(sql, input) {
|
|
18598
19324
|
const query = normalizeAiModelCallAccountingQuery(input);
|
|
19325
|
+
const sort = normalizeAiModelCallGroupSort(input.sort);
|
|
19326
|
+
const filter = pgAccountingFilterValues(query);
|
|
18599
19327
|
const parameters = [
|
|
18600
19328
|
query.projectId,
|
|
18601
19329
|
query.from.toISOString(),
|
|
18602
19330
|
query.to.toISOString(),
|
|
18603
19331
|
query.providerId ?? null,
|
|
18604
19332
|
query.modelId ?? null,
|
|
18605
|
-
query.valuationStatus ?? null
|
|
19333
|
+
query.valuationStatus ?? null,
|
|
19334
|
+
filter.kind,
|
|
19335
|
+
filter.source,
|
|
19336
|
+
filter.workflow,
|
|
19337
|
+
filter.mode,
|
|
19338
|
+
filter.id
|
|
18606
19339
|
];
|
|
18607
19340
|
const [count] = await sql.unsafe(COUNT_SQL, parameters);
|
|
18608
19341
|
const rows = await sql.unsafe(PAGE_SQL, [
|
|
18609
19342
|
...parameters,
|
|
18610
19343
|
query.limit,
|
|
18611
|
-
query.offset
|
|
19344
|
+
query.offset,
|
|
19345
|
+
sort
|
|
18612
19346
|
]);
|
|
18613
19347
|
const total = Number(count?.total);
|
|
18614
19348
|
if (!Number.isSafeInteger(total) || total < 0)
|
|
18615
19349
|
throw new RangeError("[Sixb] Invalid model-call group count.");
|
|
18616
|
-
const items = buildAiModelCallGroupsFromFragments(rows.map(aiModelCallGroupFragmentFromRow));
|
|
19350
|
+
const items = buildAiModelCallGroupsFromFragments(rows.map(aiModelCallGroupFragmentFromRow), sort);
|
|
18617
19351
|
return { items, total, hasMore: query.offset + items.length < total };
|
|
18618
19352
|
}
|
|
18619
19353
|
var TOKENLESS_KINDS = TOKENLESS_MODEL_KINDS.map((kind) => `'${kind}'`).join(", ");
|
|
18620
19354
|
var FILTERED_SQL = `
|
|
18621
19355
|
WITH filtered AS (
|
|
18622
19356
|
SELECT COALESCE(root_agent.execution_id, usage.execution_id) AS root_execution_id,
|
|
19357
|
+
-- What the call is listed under, as aiModelCallGroupKey computes it.
|
|
19358
|
+
CASE
|
|
19359
|
+
WHEN root_agent.thread_id IS NOT NULL THEN 'thread:' || root_agent.thread_id
|
|
19360
|
+
WHEN workflow_node.workflow_run_id IS NOT NULL THEN 'workflowRun:' || workflow_node.workflow_run_id
|
|
19361
|
+
ELSE COALESCE(root_agent.execution_id, usage.execution_id)
|
|
19362
|
+
END AS group_key,
|
|
18623
19363
|
usage.execution_id, usage.provider_id, usage.requested_model_id, usage.occurred_at,
|
|
18624
|
-
usage.total_tokens, usage.
|
|
19364
|
+
usage.total_tokens, usage.input_tokens, usage.cache_read_input_tokens,
|
|
19365
|
+
usage.model_kind, usage.output_images, usage.input_characters,
|
|
18625
19366
|
usage.audio_duration_ms,
|
|
18626
19367
|
execution.executor_kind, execution.executor_id, execution.authority_primitive_id AS primitive_id,
|
|
18627
19368
|
COALESCE(cost.status, 'unvalued') AS valuation_status,
|
|
@@ -18634,7 +19375,8 @@ WITH filtered AS (
|
|
|
18634
19375
|
workflow_node.node_id AS workflow_agent_step_id,
|
|
18635
19376
|
workflow_agent.node_run_id AS workflow_node_run_id,
|
|
18636
19377
|
workflow_node.workflow_id AS workflow_id,
|
|
18637
|
-
workflow_node.workflow_run_id AS workflow_run_id
|
|
19378
|
+
workflow_node.workflow_run_id AS workflow_run_id,
|
|
19379
|
+
execution.requested_by_user_id, execution.requested_by_service_account_id
|
|
18638
19380
|
FROM ai_model_call_usage AS usage
|
|
18639
19381
|
JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
|
|
18640
19382
|
LEFT JOIN agent_runs AS direct_agent
|
|
@@ -18654,23 +19396,28 @@ WITH filtered AS (
|
|
|
18654
19396
|
AND ($4::text IS NULL OR usage.provider_id = $4)
|
|
18655
19397
|
AND ($5::text IS NULL OR usage.requested_model_id = $5)
|
|
18656
19398
|
AND ($6::text IS NULL OR COALESCE(cost.status, 'unvalued') = $6)
|
|
19399
|
+
AND ${pgAccountingFilterSql(7)}
|
|
18657
19400
|
), roots AS (
|
|
18658
|
-
SELECT
|
|
18659
|
-
FROM filtered GROUP BY
|
|
19401
|
+
SELECT group_key, MAX(occurred_at) AS last_call_at, SUM(COALESCE(amount_nanos, 0)) AS cost_nanos
|
|
19402
|
+
FROM filtered GROUP BY group_key
|
|
18660
19403
|
)
|
|
18661
19404
|
`;
|
|
18662
19405
|
var COUNT_SQL = `${FILTERED_SQL} SELECT CAST(COUNT(*) AS TEXT) AS total FROM roots`;
|
|
18663
19406
|
var PAGE_SQL = `${FILTERED_SQL}, page AS (
|
|
18664
|
-
SELECT
|
|
18665
|
-
ORDER BY
|
|
19407
|
+
SELECT group_key FROM roots
|
|
19408
|
+
ORDER BY CASE WHEN $14::text = 'cost' THEN cost_nanos END DESC NULLS LAST,
|
|
19409
|
+
last_call_at DESC, group_key COLLATE "C" ASC
|
|
19410
|
+
LIMIT $12 OFFSET $13
|
|
18666
19411
|
)
|
|
18667
|
-
SELECT filtered.root_execution_id, execution_id, provider_id, requested_model_id,
|
|
19412
|
+
SELECT filtered.group_key, root_execution_id, execution_id, provider_id, requested_model_id,
|
|
18668
19413
|
MIN(occurred_at) AS first_call_at, MAX(occurred_at) AS last_call_at,
|
|
18669
19414
|
CAST(COUNT(*) AS TEXT) AS call_count,
|
|
18670
19415
|
CAST(COUNT(total_tokens) AS TEXT) AS token_count,
|
|
18671
19416
|
CAST(COUNT(CASE WHEN total_tokens IS NULL AND model_kind IN (${TOKENLESS_KINDS}) THEN 1 END) AS TEXT)
|
|
18672
19417
|
AS tokenless_count,
|
|
18673
19418
|
CAST(SUM(total_tokens) AS TEXT) AS total_tokens,
|
|
19419
|
+
CAST(SUM(input_tokens) AS TEXT) AS input_tokens,
|
|
19420
|
+
CAST(SUM(cache_read_input_tokens) AS TEXT) AS cache_read_input_tokens,
|
|
18674
19421
|
CAST(SUM(output_images) AS TEXT) AS output_images,
|
|
18675
19422
|
CAST(SUM(input_characters) AS TEXT) AS input_characters,
|
|
18676
19423
|
CAST(SUM(audio_duration_ms) AS TEXT) AS audio_duration_ms,
|
|
@@ -18687,9 +19434,11 @@ SELECT filtered.root_execution_id, execution_id, provider_id, requested_model_id
|
|
|
18687
19434
|
MAX(workflow_agent_step_id) AS workflow_agent_step_id,
|
|
18688
19435
|
MAX(workflow_node_run_id) AS workflow_node_run_id,
|
|
18689
19436
|
MAX(workflow_id) AS workflow_id,
|
|
18690
|
-
MAX(workflow_run_id) AS workflow_run_id
|
|
18691
|
-
|
|
18692
|
-
|
|
19437
|
+
MAX(workflow_run_id) AS workflow_run_id,
|
|
19438
|
+
MAX(requested_by_user_id) AS requested_by_user_id,
|
|
19439
|
+
MAX(requested_by_service_account_id) AS requested_by_service_account_id
|
|
19440
|
+
FROM filtered JOIN page ON page.group_key = filtered.group_key
|
|
19441
|
+
GROUP BY filtered.group_key, root_execution_id, execution_id, provider_id, requested_model_id,
|
|
18693
19442
|
valuation_status, amount_currency
|
|
18694
19443
|
`;
|
|
18695
19444
|
|
|
@@ -19060,6 +19809,7 @@ class PgAiCostStorage {
|
|
|
19060
19809
|
const query = normalizeAiAccountingQuery(input);
|
|
19061
19810
|
const providerId = query.providerId ?? null;
|
|
19062
19811
|
const modelId = query.modelId ?? null;
|
|
19812
|
+
const filter = pgAccountingFilterValues(query);
|
|
19063
19813
|
const rows = await this.sql`
|
|
19064
19814
|
WITH filtered AS (
|
|
19065
19815
|
SELECT usage.provider_id,
|
|
@@ -19090,7 +19840,9 @@ class PgAiCostStorage {
|
|
|
19090
19840
|
) AS attribution_workflow_id,
|
|
19091
19841
|
cost.status AS cost_status,
|
|
19092
19842
|
cost.currency AS cost_currency,
|
|
19093
|
-
cost.amount_nanos AS cost_amount_nanos
|
|
19843
|
+
cost.amount_nanos AS cost_amount_nanos,
|
|
19844
|
+
execution.requested_by_user_id AS requester_user_id,
|
|
19845
|
+
execution.requested_by_service_account_id AS requester_service_account_id
|
|
19094
19846
|
FROM ai_model_call_usage AS usage
|
|
19095
19847
|
JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
|
|
19096
19848
|
LEFT JOIN agent_runs AS direct_agent
|
|
@@ -19108,6 +19860,26 @@ class PgAiCostStorage {
|
|
|
19108
19860
|
AND usage.occurred_at >= ${query.from} AND usage.occurred_at < ${query.to}
|
|
19109
19861
|
AND (${providerId}::text IS NULL OR usage.provider_id = ${providerId})
|
|
19110
19862
|
AND (${modelId}::text IS NULL OR usage.requested_model_id = ${modelId})
|
|
19863
|
+
AND (${filter.kind}::text IS NULL OR usage.model_kind = ${filter.kind})
|
|
19864
|
+
AND (${filter.source}::text IS NULL OR (CASE
|
|
19865
|
+
WHEN direct_agent.id IS NOT NULL THEN 'agent'
|
|
19866
|
+
WHEN COALESCE(workflow_node.workflow_id,
|
|
19867
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
|
|
19868
|
+
) IS NOT NULL THEN 'workflow'
|
|
19869
|
+
WHEN execution.executor_kind = 'action' THEN 'action'
|
|
19870
|
+
ELSE 'other'
|
|
19871
|
+
END) = ${filter.source})
|
|
19872
|
+
AND (${filter.workflow}::text IS NULL OR COALESCE(workflow_node.workflow_id,
|
|
19873
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
|
|
19874
|
+
) = ${filter.workflow})
|
|
19875
|
+
AND (
|
|
19876
|
+
${filter.mode}::text IS NULL
|
|
19877
|
+
OR (${filter.mode} = 'none' AND execution.requested_by_user_id IS NULL
|
|
19878
|
+
AND execution.requested_by_service_account_id IS NULL)
|
|
19879
|
+
OR (${filter.mode} = 'user' AND execution.requested_by_user_id = ${filter.id})
|
|
19880
|
+
OR (${filter.mode} = 'serviceAccount'
|
|
19881
|
+
AND execution.requested_by_service_account_id = ${filter.id})
|
|
19882
|
+
)
|
|
19111
19883
|
), classified AS (
|
|
19112
19884
|
SELECT filtered.*,
|
|
19113
19885
|
CASE
|
|
@@ -19138,8 +19910,27 @@ class PgAiCostStorage {
|
|
|
19138
19910
|
classified.*
|
|
19139
19911
|
FROM classified
|
|
19140
19912
|
UNION ALL
|
|
19913
|
+
SELECT 'modelSeries', provider_id, requested_model_id, NULL,
|
|
19914
|
+
CASE ${input.bucket}::text
|
|
19915
|
+
WHEN 'hour' THEN
|
|
19916
|
+
date_trunc('hour', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
|
|
19917
|
+
WHEN 'day' THEN
|
|
19918
|
+
date_trunc('day', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
|
|
19919
|
+
ELSE date_trunc('week', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
|
|
19920
|
+
END,
|
|
19921
|
+
classified.*
|
|
19922
|
+
FROM classified
|
|
19923
|
+
UNION ALL
|
|
19141
19924
|
SELECT 'model', provider_id, requested_model_id, NULL, NULL, classified.* FROM classified
|
|
19142
19925
|
UNION ALL
|
|
19926
|
+
SELECT 'person',
|
|
19927
|
+
CASE
|
|
19928
|
+
WHEN requester_user_id IS NOT NULL THEN 'user'
|
|
19929
|
+
WHEN requester_service_account_id IS NOT NULL THEN 'serviceAccount'
|
|
19930
|
+
END,
|
|
19931
|
+
COALESCE(requester_user_id, requester_service_account_id), NULL, NULL, classified.*
|
|
19932
|
+
FROM classified
|
|
19933
|
+
UNION ALL
|
|
19143
19934
|
SELECT 'kind', model_kind, NULL, NULL, NULL, classified.* FROM classified
|
|
19144
19935
|
UNION ALL
|
|
19145
19936
|
SELECT 'agent', attribution_agent_kind, attribution_workflow_id, attribution_agent_step_id, NULL, classified.*
|
|
@@ -19200,9 +19991,20 @@ class PgAiCostStorage {
|
|
|
19200
19991
|
const modelId = query.modelId ?? null;
|
|
19201
19992
|
const executionId = query.executionId ?? null;
|
|
19202
19993
|
const valuationStatus = query.valuationStatus ?? null;
|
|
19994
|
+
const filter = pgAccountingFilterValues(query);
|
|
19203
19995
|
const [count] = await this.sql`
|
|
19204
19996
|
SELECT COUNT(*)::text AS total
|
|
19205
19997
|
FROM ai_model_call_usage AS usage
|
|
19998
|
+
JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
|
|
19999
|
+
LEFT JOIN agent_runs AS direct_agent
|
|
20000
|
+
ON direct_agent.project_id = usage.project_id
|
|
20001
|
+
AND direct_agent.execution_id = usage.execution_id
|
|
20002
|
+
LEFT JOIN workflow_agent_node_runs AS workflow_agent
|
|
20003
|
+
ON workflow_agent.project_id = usage.project_id
|
|
20004
|
+
AND workflow_agent.execution_id = usage.execution_id
|
|
20005
|
+
LEFT JOIN workflow_node_runs AS workflow_node
|
|
20006
|
+
ON workflow_node.project_id = workflow_agent.project_id
|
|
20007
|
+
AND workflow_node.id = workflow_agent.node_run_id
|
|
19206
20008
|
LEFT JOIN ai_model_call_valuations AS cost
|
|
19207
20009
|
ON cost.project_id = usage.project_id AND cost.usage_record_id = usage.id
|
|
19208
20010
|
WHERE usage.project_id = ${query.projectId}
|
|
@@ -19217,6 +20019,26 @@ class PgAiCostStorage {
|
|
|
19217
20019
|
ELSE 'unvalued'
|
|
19218
20020
|
END
|
|
19219
20021
|
)
|
|
20022
|
+
AND (${filter.kind}::text IS NULL OR usage.model_kind = ${filter.kind})
|
|
20023
|
+
AND (${filter.source}::text IS NULL OR (CASE
|
|
20024
|
+
WHEN direct_agent.id IS NOT NULL THEN 'agent'
|
|
20025
|
+
WHEN COALESCE(workflow_node.workflow_id,
|
|
20026
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
|
|
20027
|
+
) IS NOT NULL THEN 'workflow'
|
|
20028
|
+
WHEN execution.executor_kind = 'action' THEN 'action'
|
|
20029
|
+
ELSE 'other'
|
|
20030
|
+
END) = ${filter.source})
|
|
20031
|
+
AND (${filter.workflow}::text IS NULL OR COALESCE(workflow_node.workflow_id,
|
|
20032
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
|
|
20033
|
+
) = ${filter.workflow})
|
|
20034
|
+
AND (
|
|
20035
|
+
${filter.mode}::text IS NULL
|
|
20036
|
+
OR (${filter.mode} = 'none' AND execution.requested_by_user_id IS NULL
|
|
20037
|
+
AND execution.requested_by_service_account_id IS NULL)
|
|
20038
|
+
OR (${filter.mode} = 'user' AND execution.requested_by_user_id = ${filter.id})
|
|
20039
|
+
OR (${filter.mode} = 'serviceAccount'
|
|
20040
|
+
AND execution.requested_by_service_account_id = ${filter.id})
|
|
20041
|
+
)
|
|
19220
20042
|
`;
|
|
19221
20043
|
const rows = await this.sql`
|
|
19222
20044
|
SELECT usage.*,
|
|
@@ -19272,6 +20094,26 @@ class PgAiCostStorage {
|
|
|
19272
20094
|
ELSE 'unvalued'
|
|
19273
20095
|
END
|
|
19274
20096
|
)
|
|
20097
|
+
AND (${filter.kind}::text IS NULL OR usage.model_kind = ${filter.kind})
|
|
20098
|
+
AND (${filter.source}::text IS NULL OR (CASE
|
|
20099
|
+
WHEN direct_agent.id IS NOT NULL THEN 'agent'
|
|
20100
|
+
WHEN COALESCE(workflow_node.workflow_id,
|
|
20101
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
|
|
20102
|
+
) IS NOT NULL THEN 'workflow'
|
|
20103
|
+
WHEN execution.executor_kind = 'action' THEN 'action'
|
|
20104
|
+
ELSE 'other'
|
|
20105
|
+
END) = ${filter.source})
|
|
20106
|
+
AND (${filter.workflow}::text IS NULL OR COALESCE(workflow_node.workflow_id,
|
|
20107
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
|
|
20108
|
+
) = ${filter.workflow})
|
|
20109
|
+
AND (
|
|
20110
|
+
${filter.mode}::text IS NULL
|
|
20111
|
+
OR (${filter.mode} = 'none' AND execution.requested_by_user_id IS NULL
|
|
20112
|
+
AND execution.requested_by_service_account_id IS NULL)
|
|
20113
|
+
OR (${filter.mode} = 'user' AND execution.requested_by_user_id = ${filter.id})
|
|
20114
|
+
OR (${filter.mode} = 'serviceAccount'
|
|
20115
|
+
AND execution.requested_by_service_account_id = ${filter.id})
|
|
20116
|
+
)
|
|
19275
20117
|
ORDER BY usage.occurred_at DESC, usage.id ASC
|
|
19276
20118
|
LIMIT ${query.limit} OFFSET ${query.offset}
|
|
19277
20119
|
`;
|
|
@@ -19300,7 +20142,11 @@ async function findUsage(sql, projectId, id) {
|
|
|
19300
20142
|
return row ? usageFromRow2(row, []) : null;
|
|
19301
20143
|
}
|
|
19302
20144
|
function aggregateFragmentFromRow(row) {
|
|
19303
|
-
const dimensionKeys = row.dimension === "model" ? { providerId: required2(row.key_1), modelId: required2(row.key_2) } : row.dimension === "
|
|
20145
|
+
const dimensionKeys = row.dimension === "model" ? { providerId: required2(row.key_1), modelId: required2(row.key_2) } : row.dimension === "modelSeries" ? {
|
|
20146
|
+
providerId: required2(row.key_1),
|
|
20147
|
+
modelId: required2(row.key_2),
|
|
20148
|
+
start: new Date(required2(row.bucket_start))
|
|
20149
|
+
} : row.dimension === "person" ? row.key_1 && row.key_2 ? { requesterType: row.key_1, requesterId: row.key_2 } : {} : row.dimension === "kind" ? withModelKind(aiAccountingModelKind(row.key_1)) : row.dimension === "agent" ? {
|
|
19304
20150
|
agentKind: required2(row.key_1),
|
|
19305
20151
|
...row.key_1 === "workflowAgent" ? { workflowId: required2(row.key_2), agentStepId: required2(row.key_3) } : {}
|
|
19306
20152
|
} : row.dimension === "workflow" ? { workflowId: required2(row.key_1) } : row.dimension === "series" ? { start: new Date(required2(row.bucket_start)) } : {};
|
|
@@ -20574,11 +21420,12 @@ async function linkThread(sql, projectId, artifactId, threadId, sharedAt) {
|
|
|
20574
21420
|
import { isIP } from "net";
|
|
20575
21421
|
import { connect as connectTls } from "tls";
|
|
20576
21422
|
import postgres from "postgres";
|
|
20577
|
-
var DEFAULT_IDLE_TIMEOUT_SECONDS =
|
|
21423
|
+
var DEFAULT_IDLE_TIMEOUT_SECONDS = 60;
|
|
20578
21424
|
var DEFAULT_CONNECT_TIMEOUT_SECONDS = 10;
|
|
20579
21425
|
function createPgClient(options) {
|
|
20580
21426
|
const connection = {
|
|
20581
|
-
search_path: options.schemaName
|
|
21427
|
+
search_path: options.schemaName,
|
|
21428
|
+
enable_incremental_sort: "off"
|
|
20582
21429
|
};
|
|
20583
21430
|
if (options.statementTimeoutMillis !== undefined) {
|
|
20584
21431
|
connection.statement_timeout = options.statementTimeoutMillis;
|
|
@@ -23874,6 +24721,7 @@ class PostgresStorage {
|
|
|
23874
24721
|
schemaName;
|
|
23875
24722
|
shutdownTimeoutSeconds;
|
|
23876
24723
|
transactionScope = new AsyncLocalStorage;
|
|
24724
|
+
sortIndexes = new PgSortIndexCatalog;
|
|
23877
24725
|
constructor(options) {
|
|
23878
24726
|
if (options.maxConcurrentAggregates !== undefined && (!Number.isSafeInteger(options.maxConcurrentAggregates) || options.maxConcurrentAggregates < 1))
|
|
23879
24727
|
throw new Error("[SixbPg] maxConcurrentAggregates must be a positive integer");
|
|
@@ -23897,6 +24745,7 @@ class PostgresStorage {
|
|
|
23897
24745
|
});
|
|
23898
24746
|
this.migrators = createPostgresStorageMigrators(this.sql, this.schemaName);
|
|
23899
24747
|
const stores = createPostgresStores(this.sql, {
|
|
24748
|
+
sortIndexes: this.sortIndexes,
|
|
23900
24749
|
maxConcurrentAggregates: options.maxConcurrentAggregates,
|
|
23901
24750
|
runOntologyOperation: (run) => runPgTransaction(this.sql, run),
|
|
23902
24751
|
transactionContext: null
|
|
@@ -23970,11 +24819,19 @@ class PostgresStorage {
|
|
|
23970
24819
|
}
|
|
23971
24820
|
async prepareObjectQueries(plan) {
|
|
23972
24821
|
this.assertRootOperationAvailable();
|
|
23973
|
-
|
|
24822
|
+
try {
|
|
24823
|
+
return await preparePgObjectQueries(this.sql, this.schemaName, plan);
|
|
24824
|
+
} finally {
|
|
24825
|
+
this.sortIndexes.invalidate();
|
|
24826
|
+
}
|
|
23974
24827
|
}
|
|
23975
24828
|
async ensureObjectQueryIndexes(indexes) {
|
|
23976
24829
|
this.assertRootOperationAvailable();
|
|
23977
|
-
|
|
24830
|
+
try {
|
|
24831
|
+
return await ensurePgObjectQueryIndexes(this.sql, this.schemaName, indexes);
|
|
24832
|
+
} finally {
|
|
24833
|
+
this.sortIndexes.invalidate();
|
|
24834
|
+
}
|
|
23978
24835
|
}
|
|
23979
24836
|
async prepareObjectTextCounts(indexes) {
|
|
23980
24837
|
this.assertRootOperationAvailable();
|
|
@@ -23994,6 +24851,7 @@ class PostgresStorage {
|
|
|
23994
24851
|
createTransactionStorage(client, transactionContext) {
|
|
23995
24852
|
return {
|
|
23996
24853
|
...createPostgresStores(client, {
|
|
24854
|
+
sortIndexes: this.sortIndexes,
|
|
23997
24855
|
runOntologyOperation: async (run) => run(client),
|
|
23998
24856
|
transactionContext
|
|
23999
24857
|
}),
|
|
@@ -24012,7 +24870,7 @@ function createPostgresStores(sql, options) {
|
|
|
24012
24870
|
const shareSessions = new PgShareSessionStorage(sql);
|
|
24013
24871
|
const executions = new PgExecutionStorage(sql, auth, shareSessions);
|
|
24014
24872
|
return {
|
|
24015
|
-
objects: new PgObjectStorage(sql, options.maxConcurrentAggregates),
|
|
24873
|
+
objects: new PgObjectStorage(sql, options.sortIndexes, options.maxConcurrentAggregates),
|
|
24016
24874
|
ontology: new PgOntologyStorage({
|
|
24017
24875
|
sql,
|
|
24018
24876
|
runRootOperation: options.runOntologyOperation,
|
|
@@ -24056,4 +24914,4 @@ export {
|
|
|
24056
24914
|
migratePostgresStorage
|
|
24057
24915
|
};
|
|
24058
24916
|
|
|
24059
|
-
//# debugId=
|
|
24917
|
+
//# debugId=09BC0C242FA6748C64756E2164756E21
|