@sixb/pg 0.1.26 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -1
- package/dist/agents/agent-storage.d.ts +8 -3
- package/dist/agents/agent-storage.d.ts.map +1 -1
- package/dist/agents/computers.d.ts +22 -0
- package/dist/agents/computers.d.ts.map +1 -0
- package/dist/agents/rows.d.ts +3 -0
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/runs.d.ts.map +1 -1
- package/dist/agents/threads.d.ts +2 -1
- package/dist/agents/threads.d.ts.map +1 -1
- package/dist/agents/token-rates.d.ts +13 -0
- package/dist/agents/token-rates.d.ts.map +1 -0
- package/dist/ai-accounting-filters.d.ts +12 -0
- package/dist/ai-accounting-filters.d.ts.map +1 -0
- package/dist/ai-cost-groups.d.ts +1 -1
- package/dist/ai-cost-groups.d.ts.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +886 -158
- package/dist/index.js.map +26 -21
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/partitions.d.ts +10 -0
- package/dist/objects/partitions.d.ts.map +1 -0
- package/dist/objects/prepare-queries.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +5 -0
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/query-indexes.d.ts.map +1 -1
- package/dist/objects/sql-text.d.ts +3 -0
- package/dist/objects/sql-text.d.ts.map +1 -0
- package/dist/objects/text-count-index.d.ts.map +1 -1
- package/dist/ontology-storage/commits.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +7 -1
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +2 -0
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +2 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/pg-ai-cost-storage.d.ts.map +1 -1
- package/dist/pg-client.d.ts +1 -1
- package/dist/pg-client.d.ts.map +1 -1
- package/dist/storage-errors.d.ts +4 -1
- package/dist/storage-errors.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/agent-storage.ts +10 -3
- package/src/agents/computers.ts +95 -0
- package/src/agents/rows.ts +9 -1
- package/src/agents/runs.ts +25 -3
- package/src/agents/threads.ts +111 -20
- package/src/agents/token-rates.ts +76 -0
- package/src/ai-accounting-filters.ts +48 -0
- package/src/ai-cost-groups.ts +37 -12
- package/src/index.ts +1 -1
- package/src/migrations/084-agent-token-rates.sql +16 -0
- package/src/migrations/085-agent-computer-sessions.sql +27 -0
- package/src/migrations/086-partition-objects.sql +93 -0
- package/src/migrations.ts +8 -0
- package/src/objects/partitions.ts +163 -0
- package/src/objects/prepare-queries.ts +50 -2
- package/src/objects/query-compiler.ts +80 -66
- package/src/objects/query-indexes.ts +59 -46
- package/src/objects/sql-text.ts +8 -0
- package/src/objects/text-count-index.ts +17 -12
- package/src/ontology-storage/commits.ts +11 -2
- package/src/ontology-storage/materialization-session.ts +10 -1
- package/src/ontology-storage/materialization-writer.ts +14 -0
- package/src/ontology-storage/materializations.ts +81 -7
- package/src/pg-ai-cost-storage.ts +119 -15
- package/src/pg-client.ts +9 -2
- package/src/storage-errors.ts +28 -1
package/dist/index.js
CHANGED
|
@@ -62,6 +62,7 @@ var LOCK_TABLES = [
|
|
|
62
62
|
{ setting: "max_locks_per_transaction", table: "lock table", note: "" }
|
|
63
63
|
];
|
|
64
64
|
var OUT_OF_MEMORY = "53200";
|
|
65
|
+
var IN_FAILED_SQL_TRANSACTION = "25P02";
|
|
65
66
|
function pgErrorCode(error) {
|
|
66
67
|
if (error instanceof Error) {
|
|
67
68
|
const code = error.code;
|
|
@@ -86,6 +87,10 @@ function isRetryableTransactionConflict(error) {
|
|
|
86
87
|
const code = pgErrorCode(error);
|
|
87
88
|
return code !== undefined && RETRYABLE_TRANSACTION_CONFLICT_CODES.has(code);
|
|
88
89
|
}
|
|
90
|
+
function isLockConflict(error) {
|
|
91
|
+
const code = pgErrorCode(error);
|
|
92
|
+
return code === "55P03" || code === "40P01";
|
|
93
|
+
}
|
|
89
94
|
function isConnectionLost(error) {
|
|
90
95
|
const code = pgErrorCode(error);
|
|
91
96
|
if (code !== undefined && CONNECTION_LOST_CODES.has(code)) {
|
|
@@ -98,6 +103,9 @@ function explainPgError(error) {
|
|
|
98
103
|
if (isStorageUnavailable(error)) {
|
|
99
104
|
return storageUnavailable(error);
|
|
100
105
|
}
|
|
106
|
+
if (pgErrorCode(error) === IN_FAILED_SQL_TRANSACTION) {
|
|
107
|
+
return failedStatementLost(error);
|
|
108
|
+
}
|
|
101
109
|
if (!(error instanceof Error) || pgErrorCode(error) !== OUT_OF_MEMORY) {
|
|
102
110
|
return error;
|
|
103
111
|
}
|
|
@@ -109,6 +117,9 @@ function explainPgError(error) {
|
|
|
109
117
|
const { setting, table, note } = lockTable;
|
|
110
118
|
return new Error(`[SixbPg] PostgreSQL aborted the transaction because its shared ${table} is full ` + `(SQLSTATE 53200 "${error.message}"). The table holds ${setting} \xD7 (max_connections + ` + `max_prepared_transactions) entries shared by all sessions.${note} Raise ${setting} ` + "(applied at server restart) or shorten the transactions that overlap.", { cause: error });
|
|
111
119
|
}
|
|
120
|
+
function failedStatementLost(error) {
|
|
121
|
+
return new Error("[SixbPg] A statement was sent after an earlier statement of the same transaction failed, so " + "PostgreSQL rejected it (SQLSTATE 25P02) and the earlier error was lost. Code that handles " + "a failure inside a transaction must not send more SQL on it: let the error propagate.", { cause: error });
|
|
122
|
+
}
|
|
112
123
|
function isStorageUnavailable(error) {
|
|
113
124
|
const code = pgErrorCode(error);
|
|
114
125
|
return isConnectionLost(error) || code !== undefined && CONNECT_FAILED_CODES.has(code);
|
|
@@ -248,7 +259,10 @@ function rowToThreadRecord(row) {
|
|
|
248
259
|
sandboxParams: snapshotAgentThreadSandboxParams(typeof row.sandbox_params === "string" ? JSON.parse(row.sandbox_params) : row.sandbox_params)
|
|
249
260
|
},
|
|
250
261
|
...row.runs_on == null ? {} : { runsOn: row.runs_on },
|
|
251
|
-
...row.computer_id == null
|
|
262
|
+
...row.computer_id == null || row.computer_name == null || row.computer_folder == null ? {} : {
|
|
263
|
+
computer: { id: row.computer_id, name: row.computer_name, folder: row.computer_folder }
|
|
264
|
+
},
|
|
265
|
+
...row.moved_at == null ? {} : { movedAt: new Date(row.moved_at) },
|
|
252
266
|
status: row.status,
|
|
253
267
|
activeRunId: row.active_run_id,
|
|
254
268
|
lastMessageAt: row.last_message_at ? new Date(row.last_message_at) : undefined,
|
|
@@ -818,6 +832,73 @@ class PgAgentContextCheckpointStore {
|
|
|
818
832
|
}
|
|
819
833
|
}
|
|
820
834
|
|
|
835
|
+
// src/agents/computers.ts
|
|
836
|
+
import { assertAgentComputerSession } from "@sixb/core/internal/agent-run-storage-provider";
|
|
837
|
+
|
|
838
|
+
class PgAgentComputerSessionStore {
|
|
839
|
+
sql;
|
|
840
|
+
constructor(sql) {
|
|
841
|
+
this.sql = sql;
|
|
842
|
+
}
|
|
843
|
+
async hold(input) {
|
|
844
|
+
assertAgentComputerSession(input, "SixbPg");
|
|
845
|
+
await this.sql`
|
|
846
|
+
INSERT INTO agent_computer_sessions (
|
|
847
|
+
project_id, owner_principal_type, owner_principal_id, id,
|
|
848
|
+
computer_id, computer_name, folder, expires_at
|
|
849
|
+
) VALUES (
|
|
850
|
+
${input.projectId}, ${input.ownerPrincipal.type}, ${input.ownerPrincipal.id}, ${input.id},
|
|
851
|
+
${input.computer.id}, ${input.computer.name}, ${input.computer.folder}, ${input.expiresAt}
|
|
852
|
+
)
|
|
853
|
+
ON CONFLICT (project_id, owner_principal_type, owner_principal_id, id)
|
|
854
|
+
DO UPDATE SET
|
|
855
|
+
computer_id = excluded.computer_id,
|
|
856
|
+
computer_name = excluded.computer_name,
|
|
857
|
+
folder = excluded.folder,
|
|
858
|
+
expires_at = excluded.expires_at
|
|
859
|
+
`;
|
|
860
|
+
}
|
|
861
|
+
async release(input) {
|
|
862
|
+
const [row] = await this.sql`
|
|
863
|
+
DELETE FROM agent_computer_sessions
|
|
864
|
+
WHERE project_id = ${input.projectId}
|
|
865
|
+
AND owner_principal_type = ${input.ownerPrincipal.type}
|
|
866
|
+
AND owner_principal_id = ${input.ownerPrincipal.id}
|
|
867
|
+
AND id = ${input.id}
|
|
868
|
+
RETURNING *
|
|
869
|
+
`;
|
|
870
|
+
return row ? rowToSession(row) : null;
|
|
871
|
+
}
|
|
872
|
+
async listLive(input) {
|
|
873
|
+
const rows = await this.sql`
|
|
874
|
+
SELECT * FROM agent_computer_sessions
|
|
875
|
+
WHERE project_id = ${input.projectId}
|
|
876
|
+
AND owner_principal_type = ${input.ownerPrincipal.type}
|
|
877
|
+
AND owner_principal_id = ${input.ownerPrincipal.id}
|
|
878
|
+
AND expires_at > ${input.now}
|
|
879
|
+
ORDER BY id
|
|
880
|
+
`;
|
|
881
|
+
return rows.map(rowToSession);
|
|
882
|
+
}
|
|
883
|
+
async removeLapsed(input) {
|
|
884
|
+
const rows = await this.sql`
|
|
885
|
+
DELETE FROM agent_computer_sessions
|
|
886
|
+
WHERE project_id = ${input.projectId} AND expires_at <= ${input.now}
|
|
887
|
+
RETURNING *
|
|
888
|
+
`;
|
|
889
|
+
return rows.map(rowToSession);
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
function rowToSession(row) {
|
|
893
|
+
return {
|
|
894
|
+
projectId: row.project_id,
|
|
895
|
+
id: row.id,
|
|
896
|
+
ownerPrincipal: { type: row.owner_principal_type, id: row.owner_principal_id },
|
|
897
|
+
computer: { id: row.computer_id, name: row.computer_name, folder: row.folder },
|
|
898
|
+
expiresAt: new Date(row.expires_at)
|
|
899
|
+
};
|
|
900
|
+
}
|
|
901
|
+
|
|
821
902
|
// src/agents/memory.ts
|
|
822
903
|
import {
|
|
823
904
|
agentMemoryConflicts,
|
|
@@ -1129,7 +1210,9 @@ import {
|
|
|
1129
1210
|
assertConversationAgentRunSpec,
|
|
1130
1211
|
assertConversationThreadRow,
|
|
1131
1212
|
assertCreateSubagentRunInput,
|
|
1213
|
+
assertStartsOnComputer,
|
|
1132
1214
|
assertSubagentThreadTakesRun,
|
|
1215
|
+
assertTurnInThreadPlace,
|
|
1133
1216
|
newSubagentThreadRecord,
|
|
1134
1217
|
subagentRunMatchesCreateInput
|
|
1135
1218
|
} from "@sixb/core/internal/agent-run-storage-provider";
|
|
@@ -1217,7 +1300,7 @@ class PgAgentRunStore {
|
|
|
1217
1300
|
try {
|
|
1218
1301
|
return await runPgTransaction(this.sql, async (tx) => {
|
|
1219
1302
|
const [thread] = await tx`
|
|
1220
|
-
SELECT id, kind, active_run_id FROM agent_threads
|
|
1303
|
+
SELECT id, kind, active_run_id, runs_on FROM agent_threads
|
|
1221
1304
|
WHERE project_id = ${input.projectId} AND id = ${input.threadId}
|
|
1222
1305
|
FOR UPDATE
|
|
1223
1306
|
`;
|
|
@@ -1228,6 +1311,7 @@ class PgAgentRunStore {
|
|
|
1228
1311
|
if (thread.active_run_id !== null) {
|
|
1229
1312
|
throw new AgentStorageError4("active_run_exists", `[SixbPg] Agent thread '${input.threadId}' already has an active run '${thread.active_run_id}'.`);
|
|
1230
1313
|
}
|
|
1314
|
+
assertTurnInThreadPlace(thread.runs_on, input.spec, input.threadId, "SixbPg");
|
|
1231
1315
|
const [row] = await tx`
|
|
1232
1316
|
INSERT INTO agent_runs (
|
|
1233
1317
|
project_id,
|
|
@@ -1361,7 +1445,14 @@ class PgAgentRunStore {
|
|
|
1361
1445
|
async start(input) {
|
|
1362
1446
|
const startedAt = input.startedAt ?? new Date;
|
|
1363
1447
|
return runPgTransaction(this.sql, async (tx) => {
|
|
1364
|
-
await this.lockStatus(tx, input.projectId, input.id, "queued");
|
|
1448
|
+
const run = await this.lockStatus(tx, input.projectId, input.id, "queued");
|
|
1449
|
+
if (input.onComputer) {
|
|
1450
|
+
const [thread] = await tx`
|
|
1451
|
+
SELECT computer_id AS id, computer_folder AS folder FROM agent_threads
|
|
1452
|
+
WHERE project_id = ${input.projectId} AND id = ${run.thread_id}
|
|
1453
|
+
`;
|
|
1454
|
+
assertStartsOnComputer(rowToRunRecord(run), thread?.id ? { id: thread.id, folder: thread.folder } : undefined, input.onComputer, "SixbPg");
|
|
1455
|
+
}
|
|
1365
1456
|
const [row] = await tx`
|
|
1366
1457
|
UPDATE agent_runs
|
|
1367
1458
|
SET
|
|
@@ -1546,6 +1637,9 @@ class PgAgentRunStore {
|
|
|
1546
1637
|
whereClauses.push(`parent_run_id = $${index++}`);
|
|
1547
1638
|
params.push(input.parentRunId);
|
|
1548
1639
|
}
|
|
1640
|
+
if (input.excludeComputerTurns) {
|
|
1641
|
+
whereClauses.push("spec->>'runsOn' IS DISTINCT FROM 'computer'");
|
|
1642
|
+
}
|
|
1549
1643
|
index = appendRunListFilters(whereClauses, params, index, input, "COALESCE(started_at, created_at)");
|
|
1550
1644
|
const { rows, total, hasMore } = await queryRunList({
|
|
1551
1645
|
sql: this.sql,
|
|
@@ -1721,12 +1815,15 @@ function threadNotFound(input) {
|
|
|
1721
1815
|
|
|
1722
1816
|
// src/agents/threads.ts
|
|
1723
1817
|
import {
|
|
1818
|
+
assertAgentThreadComputer,
|
|
1724
1819
|
assertAssignableComputer,
|
|
1725
|
-
|
|
1820
|
+
assertComputerPlace,
|
|
1726
1821
|
assertConversationThreadRow as assertConversationThreadRow2,
|
|
1727
|
-
|
|
1822
|
+
assertMovableToCloud,
|
|
1823
|
+
movesComputerPlace,
|
|
1728
1824
|
participantsToAdd,
|
|
1729
1825
|
planAgentThreadFork,
|
|
1826
|
+
queuedTurnInCloud,
|
|
1730
1827
|
remapArtifactIds,
|
|
1731
1828
|
snapshotAgentThreadSandboxParams as snapshotAgentThreadSandboxParams2,
|
|
1732
1829
|
transitionAgentThreadSandbox
|
|
@@ -1756,11 +1853,14 @@ class PgAgentThreadStore {
|
|
|
1756
1853
|
});
|
|
1757
1854
|
}
|
|
1758
1855
|
async create(input) {
|
|
1759
|
-
|
|
1856
|
+
assertComputerPlace(input.runsOn, input.computer, "SixbPg");
|
|
1760
1857
|
const sandboxParams = input.sandboxParams === undefined ? null : JSON.stringify(snapshotAgentThreadSandboxParams2(input.sandboxParams));
|
|
1761
1858
|
const createdAt = input.createdAt ?? new Date;
|
|
1762
1859
|
const updatedAt = input.updatedAt ?? createdAt;
|
|
1763
|
-
const runsOn = input.runsOn === undefined ? { column: this.sql``, value: this.sql`` } : {
|
|
1860
|
+
const runsOn = input.runsOn === undefined ? { column: this.sql``, value: this.sql`` } : {
|
|
1861
|
+
column: this.sql`runs_on, computer_id, computer_name, computer_folder,`,
|
|
1862
|
+
value: this.sql`${input.runsOn}, ${input.computer?.id ?? null}, ${input.computer?.name ?? null}, ${input.computer?.folder ?? null},`
|
|
1863
|
+
};
|
|
1764
1864
|
try {
|
|
1765
1865
|
const [row] = await this.sql`
|
|
1766
1866
|
INSERT INTO agent_threads (
|
|
@@ -1800,7 +1900,7 @@ class PgAgentThreadStore {
|
|
|
1800
1900
|
try {
|
|
1801
1901
|
return await runPgTransaction(this.sql, async (tx) => {
|
|
1802
1902
|
const [source] = await tx`
|
|
1803
|
-
SELECT id, kind, active_run_id FROM agent_threads
|
|
1903
|
+
SELECT id, kind, active_run_id, runs_on, moved_at FROM agent_threads
|
|
1804
1904
|
WHERE project_id = ${input.projectId} AND id = ${input.sourceThreadId}
|
|
1805
1905
|
FOR SHARE
|
|
1806
1906
|
`;
|
|
@@ -1830,6 +1930,7 @@ class PgAgentThreadStore {
|
|
|
1830
1930
|
ORDER BY summarized_through_seq DESC, created_at DESC, id DESC
|
|
1831
1931
|
`;
|
|
1832
1932
|
const plan = planAgentThreadFork(input, {
|
|
1933
|
+
workspace: { runsOn: source.runs_on, movedAt: source.moved_at },
|
|
1833
1934
|
...activeRun?.trigger_message_id ? { triggerMessageId: activeRun.trigger_message_id } : {},
|
|
1834
1935
|
messages: messages.map((message) => ({
|
|
1835
1936
|
id: message.id,
|
|
@@ -1851,6 +1952,7 @@ class PgAgentThreadStore {
|
|
|
1851
1952
|
status,
|
|
1852
1953
|
last_message_at,
|
|
1853
1954
|
message_count,
|
|
1955
|
+
moved_at,
|
|
1854
1956
|
created_at,
|
|
1855
1957
|
updated_at
|
|
1856
1958
|
) VALUES (
|
|
@@ -1864,6 +1966,7 @@ class PgAgentThreadStore {
|
|
|
1864
1966
|
${thread.status},
|
|
1865
1967
|
${thread.lastMessageAt ?? null},
|
|
1866
1968
|
${thread.messageCount},
|
|
1969
|
+
${thread.movedAt ?? null},
|
|
1867
1970
|
${thread.createdAt},
|
|
1868
1971
|
${thread.updatedAt}
|
|
1869
1972
|
)
|
|
@@ -1984,6 +2087,10 @@ class PgAgentThreadStore {
|
|
|
1984
2087
|
)`);
|
|
1985
2088
|
params.push(input.participantPrincipal.type, input.participantPrincipal.id);
|
|
1986
2089
|
}
|
|
2090
|
+
if (input.activeOnComputer) {
|
|
2091
|
+
whereClauses.push(`computer_id = $${index++} AND computer_folder = $${index++} AND active_run_id IS NOT NULL`);
|
|
2092
|
+
params.push(input.activeOnComputer.id, input.activeOnComputer.folder);
|
|
2093
|
+
}
|
|
1987
2094
|
const dir = input.order === "asc" ? "ASC" : "DESC";
|
|
1988
2095
|
const { rows, total, hasMore } = await queryAgentList({
|
|
1989
2096
|
sql: this.sql,
|
|
@@ -2047,25 +2154,77 @@ class PgAgentThreadStore {
|
|
|
2047
2154
|
return rows.length > 0;
|
|
2048
2155
|
}
|
|
2049
2156
|
async assignComputer(input) {
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2157
|
+
assertAgentThreadComputer(input.computer, "SixbPg");
|
|
2158
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
2159
|
+
const { thread, active } = await lockThreadAndActiveRun(tx, input);
|
|
2160
|
+
assertAssignableComputer(thread, active, input.computer, input.id, "SixbPg");
|
|
2161
|
+
const movedAt = movesComputerPlace(thread, input.computer) ? input.movedAt ?? new Date : null;
|
|
2162
|
+
const [assigned] = await tx`
|
|
2163
|
+
UPDATE agent_threads
|
|
2164
|
+
SET computer_id = ${input.computer.id},
|
|
2165
|
+
computer_name = ${input.computer.name},
|
|
2166
|
+
computer_folder = ${input.computer.folder},
|
|
2167
|
+
moved_at = COALESCE(${movedAt}::timestamptz, moved_at)
|
|
2168
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
2169
|
+
RETURNING *
|
|
2170
|
+
`;
|
|
2171
|
+
return rowToThreadRecord(assigned);
|
|
2172
|
+
});
|
|
2173
|
+
}
|
|
2174
|
+
async moveToCloud(input) {
|
|
2175
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
2176
|
+
const { thread, active } = await lockThreadAndActiveRun(tx, input);
|
|
2177
|
+
assertMovableToCloud(thread, active, input.id, "SixbPg");
|
|
2178
|
+
const queued = queuedTurnInCloud(active);
|
|
2179
|
+
if (queued) {
|
|
2180
|
+
await tx`
|
|
2181
|
+
UPDATE agent_runs SET spec = ${JSON.stringify(queued.spec)}::text::jsonb
|
|
2182
|
+
WHERE project_id = ${input.projectId} AND id = ${queued.id}
|
|
2183
|
+
`;
|
|
2184
|
+
}
|
|
2185
|
+
const [movedThread] = await tx`
|
|
2186
|
+
UPDATE agent_threads
|
|
2187
|
+
SET runs_on = NULL, computer_id = NULL, computer_name = NULL, computer_folder = NULL,
|
|
2188
|
+
moved_at = ${input.movedAt ?? new Date}
|
|
2189
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
2190
|
+
RETURNING *
|
|
2191
|
+
`;
|
|
2192
|
+
return {
|
|
2193
|
+
thread: rowToThreadRecord(movedThread),
|
|
2194
|
+
...queued ? { queued } : {}
|
|
2195
|
+
};
|
|
2196
|
+
});
|
|
2064
2197
|
}
|
|
2065
2198
|
async listParticipants(input) {
|
|
2066
2199
|
return selectParticipants(this.sql, input.projectId, [...new Set(input.threadIds)]);
|
|
2067
2200
|
}
|
|
2068
2201
|
}
|
|
2202
|
+
async function lockThreadAndActiveRun(tx, input) {
|
|
2203
|
+
for (let attempt = 0;; attempt += 1) {
|
|
2204
|
+
const [peek] = await tx`
|
|
2205
|
+
SELECT active_run_id FROM agent_threads
|
|
2206
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
2207
|
+
`;
|
|
2208
|
+
const runId = peek?.active_run_id ?? null;
|
|
2209
|
+
const runs = runId ? await tx`
|
|
2210
|
+
SELECT * FROM agent_runs WHERE project_id = ${input.projectId} AND id = ${runId}
|
|
2211
|
+
FOR UPDATE
|
|
2212
|
+
` : [];
|
|
2213
|
+
const [row] = await tx`
|
|
2214
|
+
SELECT * FROM agent_threads WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
2215
|
+
FOR UPDATE
|
|
2216
|
+
`;
|
|
2217
|
+
if ((row?.active_run_id ?? null) === runId) {
|
|
2218
|
+
return {
|
|
2219
|
+
thread: row ? rowToThreadRecord(row) : null,
|
|
2220
|
+
active: runs[0] ? rowToRunRecord(runs[0]) : null
|
|
2221
|
+
};
|
|
2222
|
+
}
|
|
2223
|
+
if (attempt >= 3) {
|
|
2224
|
+
throw new AgentStorageError6("invalid_state", `[SixbPg] Agent thread '${input.id}' kept changing turns while it moved: try again.`);
|
|
2225
|
+
}
|
|
2226
|
+
}
|
|
2227
|
+
}
|
|
2069
2228
|
async function selectParticipants(sql, projectId, threadIds) {
|
|
2070
2229
|
if (threadIds.length === 0) {
|
|
2071
2230
|
return [];
|
|
@@ -2130,6 +2289,65 @@ async function remapCopiedParts(sql, projectId, messageIds, copies) {
|
|
|
2130
2289
|
}
|
|
2131
2290
|
}
|
|
2132
2291
|
|
|
2292
|
+
// src/agents/token-rates.ts
|
|
2293
|
+
import {
|
|
2294
|
+
AGENT_TOKEN_RATE_SAMPLES,
|
|
2295
|
+
assertAgentTokenRateRecord
|
|
2296
|
+
} from "@sixb/core/internal/agent-run-storage-provider";
|
|
2297
|
+
class PgAgentTokenRateStore {
|
|
2298
|
+
sql;
|
|
2299
|
+
constructor(sql) {
|
|
2300
|
+
this.sql = sql;
|
|
2301
|
+
}
|
|
2302
|
+
async record(input) {
|
|
2303
|
+
assertAgentTokenRateRecord(input, "SixbPg");
|
|
2304
|
+
await runPgTransaction(this.sql, async (tx) => {
|
|
2305
|
+
await tx`
|
|
2306
|
+
INSERT INTO agent_token_rates (
|
|
2307
|
+
project_id, run_id, provider_id, model_id, bytes, tokens, measured_at
|
|
2308
|
+
) VALUES (
|
|
2309
|
+
${input.projectId}, ${input.runId}, ${input.providerId}, ${input.modelId},
|
|
2310
|
+
${input.bytes}, ${input.tokens}, ${input.measuredAt}
|
|
2311
|
+
)
|
|
2312
|
+
ON CONFLICT (project_id, run_id) DO UPDATE SET
|
|
2313
|
+
provider_id = EXCLUDED.provider_id, model_id = EXCLUDED.model_id,
|
|
2314
|
+
bytes = EXCLUDED.bytes, tokens = EXCLUDED.tokens, measured_at = EXCLUDED.measured_at
|
|
2315
|
+
`;
|
|
2316
|
+
await tx`
|
|
2317
|
+
DELETE FROM agent_token_rates
|
|
2318
|
+
WHERE project_id = ${input.projectId} AND provider_id = ${input.providerId}
|
|
2319
|
+
AND model_id = ${input.modelId}
|
|
2320
|
+
AND run_id NOT IN (
|
|
2321
|
+
SELECT run_id FROM agent_token_rates
|
|
2322
|
+
WHERE project_id = ${input.projectId} AND provider_id = ${input.providerId}
|
|
2323
|
+
AND model_id = ${input.modelId}
|
|
2324
|
+
ORDER BY measured_at DESC, run_id DESC
|
|
2325
|
+
LIMIT ${AGENT_TOKEN_RATE_SAMPLES}
|
|
2326
|
+
)
|
|
2327
|
+
`;
|
|
2328
|
+
});
|
|
2329
|
+
}
|
|
2330
|
+
async listLatest(input) {
|
|
2331
|
+
const rows = await this.sql`
|
|
2332
|
+
SELECT project_id, run_id, provider_id, model_id, bytes, tokens, measured_at
|
|
2333
|
+
FROM agent_token_rates
|
|
2334
|
+
WHERE project_id = ${input.projectId} AND provider_id = ${input.providerId}
|
|
2335
|
+
AND model_id = ${input.modelId}
|
|
2336
|
+
ORDER BY measured_at DESC, run_id DESC
|
|
2337
|
+
LIMIT ${AGENT_TOKEN_RATE_SAMPLES}
|
|
2338
|
+
`;
|
|
2339
|
+
return rows.map((row) => ({
|
|
2340
|
+
projectId: row.project_id,
|
|
2341
|
+
providerId: row.provider_id,
|
|
2342
|
+
modelId: row.model_id,
|
|
2343
|
+
runId: row.run_id,
|
|
2344
|
+
bytes: row.bytes,
|
|
2345
|
+
tokens: row.tokens,
|
|
2346
|
+
measuredAt: row.measured_at
|
|
2347
|
+
}));
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
|
|
2133
2351
|
// src/agents/agent-storage.ts
|
|
2134
2352
|
class PgAgentStorage {
|
|
2135
2353
|
threads;
|
|
@@ -2139,6 +2357,8 @@ class PgAgentStorage {
|
|
|
2139
2357
|
memory;
|
|
2140
2358
|
approvals;
|
|
2141
2359
|
approvalModes;
|
|
2360
|
+
computers;
|
|
2361
|
+
tokenRates;
|
|
2142
2362
|
constructor(options) {
|
|
2143
2363
|
this.threads = new PgAgentThreadStore(options.sql);
|
|
2144
2364
|
this.runs = new PgAgentRunStore(options.sql, options.executions);
|
|
@@ -2147,6 +2367,8 @@ class PgAgentStorage {
|
|
|
2147
2367
|
this.memory = new PgAgentMemoryStore(options.sql);
|
|
2148
2368
|
this.approvals = new PgAgentApprovalStore(options.sql);
|
|
2149
2369
|
this.approvalModes = new PgAgentApprovalModeStore(options.sql);
|
|
2370
|
+
this.computers = new PgAgentComputerSessionStore(options.sql);
|
|
2371
|
+
this.tokenRates = new PgAgentTokenRateStore(options.sql);
|
|
2150
2372
|
}
|
|
2151
2373
|
}
|
|
2152
2374
|
// src/auth-storage/auth-storage.ts
|
|
@@ -9801,6 +10023,151 @@ ALTER TABLE agent_threads ADD COLUMN runs_on TEXT CHECK (runs_on IS NULL OR runs
|
|
|
9801
10023
|
ALTER TABLE agent_threads ADD COLUMN computer_id TEXT;
|
|
9802
10024
|
`;
|
|
9803
10025
|
|
|
10026
|
+
// src/migrations/084-agent-token-rates.sql
|
|
10027
|
+
var _084_agent_token_rates_default = `-- What each model's tokenizer measured on a project's agent runs, one row per run: the bytes of a
|
|
10028
|
+
-- request's text as the Agent's context estimator weighs them, against the text tokens the
|
|
10029
|
+
-- provider counted. A record deletes the model's rows past its newest 20, so the table stays small.
|
|
10030
|
+
CREATE TABLE agent_token_rates (
|
|
10031
|
+
project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
|
|
10032
|
+
run_id TEXT NOT NULL CHECK (length(trim(run_id)) > 0),
|
|
10033
|
+
provider_id TEXT NOT NULL CHECK (length(trim(provider_id)) > 0),
|
|
10034
|
+
model_id TEXT NOT NULL CHECK (length(trim(model_id)) > 0),
|
|
10035
|
+
bytes INTEGER NOT NULL CHECK (bytes > 0),
|
|
10036
|
+
tokens INTEGER NOT NULL CHECK (tokens > 0),
|
|
10037
|
+
measured_at TIMESTAMPTZ NOT NULL,
|
|
10038
|
+
PRIMARY KEY (project_id, run_id)
|
|
10039
|
+
);
|
|
10040
|
+
|
|
10041
|
+
CREATE INDEX idx_agent_token_rates_model
|
|
10042
|
+
ON agent_token_rates (project_id, provider_id, model_id, measured_at DESC, run_id DESC);
|
|
10043
|
+
`;
|
|
10044
|
+
|
|
10045
|
+
// src/migrations/085-agent-computer-sessions.sql
|
|
10046
|
+
var _085_agent_computer_sessions_default = `-- Where a computer's thread runs: the computer's name and the folder there, beside its id. A
|
|
10047
|
+
-- computer kept before folders were has none until its owner moves the thread.
|
|
10048
|
+
ALTER TABLE agent_threads ADD COLUMN computer_name TEXT;
|
|
10049
|
+
ALTER TABLE agent_threads ADD COLUMN computer_folder TEXT;
|
|
10050
|
+
UPDATE agent_threads SET computer_id = NULL WHERE computer_id IS NOT NULL;
|
|
10051
|
+
-- When its turns last moved to where they run now: the files earlier turns used are not there.
|
|
10052
|
+
ALTER TABLE agent_threads ADD COLUMN moved_at TIMESTAMPTZ;
|
|
10053
|
+
|
|
10054
|
+
-- Each open sixb on its owner's computers, a lease it renews: while one is live, the threads that
|
|
10055
|
+
-- run on its computer in its folder are reachable there.
|
|
10056
|
+
CREATE TABLE agent_computer_sessions (
|
|
10057
|
+
project_id TEXT NOT NULL,
|
|
10058
|
+
owner_principal_type TEXT NOT NULL CHECK (owner_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
10059
|
+
owner_principal_id TEXT NOT NULL CHECK (length(owner_principal_id) > 0),
|
|
10060
|
+
id TEXT NOT NULL CHECK (length(id) > 0),
|
|
10061
|
+
computer_id TEXT NOT NULL,
|
|
10062
|
+
computer_name TEXT NOT NULL,
|
|
10063
|
+
folder TEXT NOT NULL,
|
|
10064
|
+
expires_at TIMESTAMPTZ NOT NULL,
|
|
10065
|
+
PRIMARY KEY (project_id, owner_principal_type, owner_principal_id, id)
|
|
10066
|
+
);
|
|
10067
|
+
CREATE INDEX agent_computer_sessions_by_expiry ON agent_computer_sessions (project_id, expires_at);
|
|
10068
|
+
|
|
10069
|
+
-- Each open sixb asks every 20 seconds which of its owner's threads here have a turn active.
|
|
10070
|
+
CREATE INDEX agent_threads_active_on_computer
|
|
10071
|
+
ON agent_threads (project_id, owner_principal_type, owner_principal_id, computer_id, computer_folder)
|
|
10072
|
+
WHERE active_run_id IS NOT NULL;
|
|
10073
|
+
`;
|
|
10074
|
+
|
|
10075
|
+
// src/migrations/086-partition-objects.sql
|
|
10076
|
+
var _086_partition_objects_default = `-- Partition objects by object type, so a query plans against its own type's indexes alone instead
|
|
10077
|
+
-- of every index of every type. Query indexes and extended statistics go with the old table:
|
|
10078
|
+
-- preparation builds them again on the new partitions, right after this migration.
|
|
10079
|
+
DO $$
|
|
10080
|
+
DECLARE
|
|
10081
|
+
item record;
|
|
10082
|
+
columns_list text;
|
|
10083
|
+
restore_constraints text[] := '{}';
|
|
10084
|
+
restore_indexes text[] := '{}';
|
|
10085
|
+
statement text;
|
|
10086
|
+
BEGIN
|
|
10087
|
+
-- Copying the rows takes as long as it takes, whatever timeout this role's queries run under.
|
|
10088
|
+
-- Waiting for the lock does not: while this waits, every other query waits behind it, so it gives
|
|
10089
|
+
-- way to a long query, and the migration fails cleanly if that query outlasts every attempt.
|
|
10090
|
+
PERFORM set_config('statement_timeout', '0', true);
|
|
10091
|
+
PERFORM set_config('lock_timeout', '3s', true);
|
|
10092
|
+
FOR attempt IN 1..5 LOOP
|
|
10093
|
+
BEGIN
|
|
10094
|
+
LOCK TABLE objects IN ACCESS EXCLUSIVE MODE;
|
|
10095
|
+
EXIT;
|
|
10096
|
+
EXCEPTION WHEN lock_not_available THEN
|
|
10097
|
+
IF attempt = 5 THEN RAISE; END IF;
|
|
10098
|
+
PERFORM pg_sleep(attempt * 0.5);
|
|
10099
|
+
END;
|
|
10100
|
+
END LOOP;
|
|
10101
|
+
|
|
10102
|
+
-- objects is held now. Tables that reference it may have readers, so wait for those a little
|
|
10103
|
+
-- longer, and fail cleanly rather than hold every query behind objects for good.
|
|
10104
|
+
PERFORM set_config('lock_timeout', '30s', true);
|
|
10105
|
+
|
|
10106
|
+
FOR item IN
|
|
10107
|
+
SELECT conrelid::regclass::text AS table_name, conname, pg_get_constraintdef(oid) AS definition
|
|
10108
|
+
FROM pg_constraint
|
|
10109
|
+
WHERE confrelid = 'objects'::regclass AND contype = 'f' AND conparentid = 0
|
|
10110
|
+
LOOP
|
|
10111
|
+
EXECUTE format('ALTER TABLE %s DROP CONSTRAINT %I', item.table_name, item.conname);
|
|
10112
|
+
restore_constraints := restore_constraints || format(
|
|
10113
|
+
'ALTER TABLE %s ADD CONSTRAINT %I %s', item.table_name, item.conname, item.definition
|
|
10114
|
+
);
|
|
10115
|
+
END LOOP;
|
|
10116
|
+
|
|
10117
|
+
SELECT string_agg(quote_ident(attname), ', ' ORDER BY attnum) INTO columns_list
|
|
10118
|
+
FROM pg_attribute
|
|
10119
|
+
WHERE attrelid = 'objects'::regclass AND attnum > 0 AND NOT attisdropped AND attgenerated = '';
|
|
10120
|
+
|
|
10121
|
+
-- The table's general indexes stay on the parent under their names; prepared ones do not.
|
|
10122
|
+
FOR item IN
|
|
10123
|
+
SELECT c.relname AS name, pg_get_indexdef(c.oid) AS definition
|
|
10124
|
+
FROM pg_index i
|
|
10125
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
10126
|
+
WHERE i.indrelid = 'objects'::regclass
|
|
10127
|
+
AND NOT i.indisprimary
|
|
10128
|
+
AND c.relname NOT LIKE 'sixb\\_query\\_%'
|
|
10129
|
+
AND c.relname NOT LIKE 'sixb\\_text\\_count\\_%'
|
|
10130
|
+
LOOP
|
|
10131
|
+
restore_indexes := restore_indexes || item.definition;
|
|
10132
|
+
EXECUTE format('DROP INDEX %I', item.name);
|
|
10133
|
+
END LOOP;
|
|
10134
|
+
|
|
10135
|
+
ALTER TABLE objects RENAME TO objects_unpartitioned;
|
|
10136
|
+
ALTER INDEX objects_pkey RENAME TO objects_unpartitioned_pkey;
|
|
10137
|
+
|
|
10138
|
+
CREATE TABLE objects (
|
|
10139
|
+
LIKE objects_unpartitioned
|
|
10140
|
+
INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE INCLUDING COMMENTS,
|
|
10141
|
+
PRIMARY KEY (project_id, object_type_id, primary_id)
|
|
10142
|
+
) PARTITION BY LIST (object_type_id);
|
|
10143
|
+
|
|
10144
|
+
-- Holds a type until its first preparation gives it a partition.
|
|
10145
|
+
CREATE TABLE objects_default PARTITION OF objects DEFAULT;
|
|
10146
|
+
|
|
10147
|
+
FOR item IN SELECT DISTINCT object_type_id AS id FROM objects_unpartitioned LOOP
|
|
10148
|
+
EXECUTE format(
|
|
10149
|
+
'CREATE TABLE %I PARTITION OF objects FOR VALUES IN (%L)',
|
|
10150
|
+
'objects_' || left(md5(item.id), 24),
|
|
10151
|
+
item.id
|
|
10152
|
+
);
|
|
10153
|
+
END LOOP;
|
|
10154
|
+
|
|
10155
|
+
EXECUTE format(
|
|
10156
|
+
'INSERT INTO objects (%s) SELECT %s FROM objects_unpartitioned', columns_list, columns_list
|
|
10157
|
+
);
|
|
10158
|
+
DROP TABLE objects_unpartitioned;
|
|
10159
|
+
|
|
10160
|
+
FOREACH statement IN ARRAY restore_indexes LOOP
|
|
10161
|
+
EXECUTE statement;
|
|
10162
|
+
END LOOP;
|
|
10163
|
+
FOREACH statement IN ARRAY restore_constraints LOOP
|
|
10164
|
+
EXECUTE statement;
|
|
10165
|
+
END LOOP;
|
|
10166
|
+
END $$;
|
|
10167
|
+
|
|
10168
|
+
ANALYZE objects;
|
|
10169
|
+
`;
|
|
10170
|
+
|
|
9804
10171
|
// src/migrations.ts
|
|
9805
10172
|
var POSTGRES_STORAGE_ADAPTER_ID = "SixbPostgresStorage";
|
|
9806
10173
|
function pgSql(id, sqlText) {
|
|
@@ -10050,13 +10417,122 @@ var postgresStorageMigrations = defineMigrations({
|
|
|
10050
10417
|
pgSql("080-subagent-threads", _080_subagent_threads_default),
|
|
10051
10418
|
pgSql("081-subagent-questions", _081_subagent_questions_default),
|
|
10052
10419
|
pgSql("082-agent-run-cancel-requests", _082_agent_run_cancel_requests_default),
|
|
10053
|
-
pgSql("083-agent-thread-runs-on", _083_agent_thread_runs_on_default)
|
|
10420
|
+
pgSql("083-agent-thread-runs-on", _083_agent_thread_runs_on_default),
|
|
10421
|
+
pgSql("084-agent-token-rates", _084_agent_token_rates_default),
|
|
10422
|
+
pgSql("085-agent-computer-sessions", _085_agent_computer_sessions_default),
|
|
10423
|
+
pgSql("086-partition-objects", _086_partition_objects_default)
|
|
10054
10424
|
]
|
|
10055
10425
|
});
|
|
10056
10426
|
|
|
10057
10427
|
// src/objects/query-indexes.ts
|
|
10428
|
+
import { createHash as createHash4 } from "crypto";
|
|
10429
|
+
import { linkEndpointObjectTypeIds as linkEndpointObjectTypeIds2 } from "@sixb/core/internal/query";
|
|
10430
|
+
|
|
10431
|
+
// src/objects/partitions.ts
|
|
10058
10432
|
import { createHash as createHash3 } from "crypto";
|
|
10059
10433
|
|
|
10434
|
+
// src/objects/sql-text.ts
|
|
10435
|
+
function identifier(value) {
|
|
10436
|
+
return `"${value.replace(/"/g, '""')}"`;
|
|
10437
|
+
}
|
|
10438
|
+
function literal(value) {
|
|
10439
|
+
if (value.includes("\x00"))
|
|
10440
|
+
throw new Error("[SixbPg] SQL literals cannot contain NUL");
|
|
10441
|
+
return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`;
|
|
10442
|
+
}
|
|
10443
|
+
|
|
10444
|
+
// src/objects/partitions.ts
|
|
10445
|
+
var LOCK_TIMEOUT = "1s";
|
|
10446
|
+
var LOCK_ATTEMPTS = 120;
|
|
10447
|
+
var LOCK_RETRY_MS = 1000;
|
|
10448
|
+
function objectPartitionName(objectTypeId) {
|
|
10449
|
+
return `objects_${createHash3("md5").update(objectTypeId).digest("hex").slice(0, 24)}`;
|
|
10450
|
+
}
|
|
10451
|
+
async function ensureObjectPartition(connection, schema, objectTypeId) {
|
|
10452
|
+
const name = objectPartitionName(objectTypeId);
|
|
10453
|
+
const partition = `${identifier(schema)}.${identifier(name)}`;
|
|
10454
|
+
const [existing] = await connection`
|
|
10455
|
+
SELECT to_regclass(${partition})::text AS found`;
|
|
10456
|
+
if (existing?.found)
|
|
10457
|
+
return name;
|
|
10458
|
+
for (let attempt = 1;; attempt++) {
|
|
10459
|
+
try {
|
|
10460
|
+
await createObjectPartition(connection, schema, partition, objectTypeId);
|
|
10461
|
+
return name;
|
|
10462
|
+
} catch (error) {
|
|
10463
|
+
if (!isLockConflict(error))
|
|
10464
|
+
throw error;
|
|
10465
|
+
if (attempt === LOCK_ATTEMPTS) {
|
|
10466
|
+
throw new Error(`[SixbPg] Could not add a partition for object type '${objectTypeId}': other queries held their locks through ${LOCK_ATTEMPTS} attempts. Retry when they finish.`, { cause: error });
|
|
10467
|
+
}
|
|
10468
|
+
}
|
|
10469
|
+
await Bun.sleep(LOCK_RETRY_MS);
|
|
10470
|
+
}
|
|
10471
|
+
}
|
|
10472
|
+
async function createObjectPartition(connection, schema, partition, objectTypeId) {
|
|
10473
|
+
const parent = `${identifier(schema)}.objects`;
|
|
10474
|
+
const fallback = `${identifier(schema)}.objects_default`;
|
|
10475
|
+
const type = literal(objectTypeId);
|
|
10476
|
+
const hasRows = async () => {
|
|
10477
|
+
const [row] = await connection.unsafe(`SELECT 1 FROM ${fallback} WHERE object_type_id = ${type} LIMIT 1`);
|
|
10478
|
+
return row !== undefined;
|
|
10479
|
+
};
|
|
10480
|
+
const guarded = (run) => runPgTransactionOn(connection, async () => {
|
|
10481
|
+
await connection.unsafe(`SET LOCAL lock_timeout = '${LOCK_TIMEOUT}'`);
|
|
10482
|
+
await connection.unsafe("SET LOCAL statement_timeout = 0");
|
|
10483
|
+
await run();
|
|
10484
|
+
});
|
|
10485
|
+
let moving = false;
|
|
10486
|
+
await guarded(async () => {
|
|
10487
|
+
moving = await hasRows();
|
|
10488
|
+
});
|
|
10489
|
+
await guarded(async () => {
|
|
10490
|
+
const references = await referencesToObjects(connection, parent);
|
|
10491
|
+
await connection.unsafe(`LOCK TABLE ONLY ${parent} IN ${moving ? "ACCESS EXCLUSIVE" : "SHARE UPDATE EXCLUSIVE"} MODE`);
|
|
10492
|
+
for (const { table_name: table } of references) {
|
|
10493
|
+
await connection.unsafe(`LOCK TABLE ${table} IN SHARE ROW EXCLUSIVE MODE`);
|
|
10494
|
+
}
|
|
10495
|
+
await connection.unsafe(`LOCK TABLE ${fallback} IN ACCESS EXCLUSIVE MODE`);
|
|
10496
|
+
if (!moving && await hasRows()) {
|
|
10497
|
+
throw Object.assign(new Error("[SixbPg] A row arrived while adding its partition."), {
|
|
10498
|
+
code: "55P03"
|
|
10499
|
+
});
|
|
10500
|
+
}
|
|
10501
|
+
await connection.unsafe(`CREATE TABLE ${partition} (LIKE ${parent} INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE)`);
|
|
10502
|
+
if (moving)
|
|
10503
|
+
await moveRowsFromDefault(connection, parent, fallback, partition, type, references);
|
|
10504
|
+
await connection.unsafe(`ALTER TABLE ${parent} ATTACH PARTITION ${partition} FOR VALUES IN (${type})`);
|
|
10505
|
+
if (moving) {
|
|
10506
|
+
for (const reference of references) {
|
|
10507
|
+
await connection.unsafe(`ALTER TABLE ${reference.table_name} ADD CONSTRAINT ${identifier(reference.name)} ${reference.definition}`);
|
|
10508
|
+
}
|
|
10509
|
+
}
|
|
10510
|
+
});
|
|
10511
|
+
if (moving)
|
|
10512
|
+
await guarded(() => connection.unsafe(`ANALYZE ${partition}`).then(() => {
|
|
10513
|
+
return;
|
|
10514
|
+
}));
|
|
10515
|
+
}
|
|
10516
|
+
function referencesToObjects(connection, parent) {
|
|
10517
|
+
return connection`
|
|
10518
|
+
SELECT conrelid::regclass::text AS table_name, conname AS name,
|
|
10519
|
+
pg_get_constraintdef(oid) AS definition
|
|
10520
|
+
FROM pg_constraint
|
|
10521
|
+
WHERE confrelid = ${parent}::regclass AND contype = 'f' AND conparentid = 0`;
|
|
10522
|
+
}
|
|
10523
|
+
async function moveRowsFromDefault(connection, parent, fallback, partition, type, references) {
|
|
10524
|
+
for (const reference of references) {
|
|
10525
|
+
await connection.unsafe(`ALTER TABLE ${reference.table_name} DROP CONSTRAINT ${identifier(reference.name)}`);
|
|
10526
|
+
}
|
|
10527
|
+
const columns = await connection`
|
|
10528
|
+
SELECT attname AS name FROM pg_attribute
|
|
10529
|
+
WHERE attrelid = ${parent}::regclass AND attnum > 0 AND NOT attisdropped AND attgenerated = ''
|
|
10530
|
+
ORDER BY attnum`;
|
|
10531
|
+
const list = columns.map((column) => identifier(column.name)).join(", ");
|
|
10532
|
+
await connection.unsafe(`INSERT INTO ${partition} (${list}) SELECT ${list} FROM ${fallback} WHERE object_type_id = ${type}`);
|
|
10533
|
+
await connection.unsafe(`DELETE FROM ${fallback} WHERE object_type_id = ${type}`);
|
|
10534
|
+
}
|
|
10535
|
+
|
|
10060
10536
|
// src/objects/preparation-lock.ts
|
|
10061
10537
|
async function lockPgPreparation(sql, key) {
|
|
10062
10538
|
let delay = 25;
|
|
@@ -10077,6 +10553,7 @@ import {
|
|
|
10077
10553
|
import {
|
|
10078
10554
|
hasVectorProfile,
|
|
10079
10555
|
isVectorProfileQuery,
|
|
10556
|
+
linkEndpointObjectTypeIds,
|
|
10080
10557
|
normalizeVector,
|
|
10081
10558
|
vectorSearchCandidateLimit
|
|
10082
10559
|
} from "@sixb/core/internal/query";
|
|
@@ -10227,7 +10704,7 @@ function compileObjectQueryInternal(projectId, query, ctx) {
|
|
|
10227
10704
|
case "page":
|
|
10228
10705
|
return compilePage(projectId, query.input, query.pageSize, query.pageToken, ctx);
|
|
10229
10706
|
case "traverse":
|
|
10230
|
-
return compileTraversal(projectId, query
|
|
10707
|
+
return compileTraversal(projectId, query, ctx);
|
|
10231
10708
|
case "set":
|
|
10232
10709
|
return compileSet(projectId, query.op, query.inputs, ctx);
|
|
10233
10710
|
case "project":
|
|
@@ -10295,7 +10772,7 @@ function compileStart(projectId, query, ctx) {
|
|
|
10295
10772
|
const order = compileOrder(identityOrderFields());
|
|
10296
10773
|
const objects = pgObjectsOfType(ctx.source, query.objectTypeId);
|
|
10297
10774
|
const sql = `
|
|
10298
|
-
SELECT
|
|
10775
|
+
SELECT *
|
|
10299
10776
|
FROM ${objects.table}
|
|
10300
10777
|
WHERE project_id = ? AND object_type_id = ? ${objects.filter}
|
|
10301
10778
|
${ctx.startPredicate ? `AND (${ctx.startPredicate})` : ""}
|
|
@@ -10333,7 +10810,7 @@ function compileRefs(projectId, query, ctx) {
|
|
|
10333
10810
|
FROM jsonb_array_elements(?::text::jsonb) AS ref(value)
|
|
10334
10811
|
`;
|
|
10335
10812
|
const sql = `
|
|
10336
|
-
SELECT selected
|
|
10813
|
+
SELECT selected.*
|
|
10337
10814
|
FROM (${requested}) AS requested
|
|
10338
10815
|
JOIN ${ctx.source.objectsTable} AS selected
|
|
10339
10816
|
ON selected.project_id = ?
|
|
@@ -10412,7 +10889,6 @@ function compileSort(projectId, inputQuery, fields, ctx) {
|
|
|
10412
10889
|
input.object_type_id,
|
|
10413
10890
|
input.primary_id,
|
|
10414
10891
|
input.properties,
|
|
10415
|
-
input.properties AS _cursor_properties,
|
|
10416
10892
|
input.created_at,
|
|
10417
10893
|
input.updated_at,
|
|
10418
10894
|
input.version,
|
|
@@ -10506,17 +10982,17 @@ function compilePage(projectId, inputQuery, rawPageSize, pageToken, ctx) {
|
|
|
10506
10982
|
}
|
|
10507
10983
|
};
|
|
10508
10984
|
}
|
|
10509
|
-
function compileTraversal(projectId,
|
|
10510
|
-
const input = compileObjectQueryInternal(projectId,
|
|
10985
|
+
function compileTraversal(projectId, traversal, ctx) {
|
|
10986
|
+
const input = compileObjectQueryInternal(projectId, traversal.input, {
|
|
10511
10987
|
...exactContext(ctx),
|
|
10512
10988
|
unordered: true,
|
|
10513
10989
|
startPredicate: undefined,
|
|
10514
10990
|
isolateSortOverflow: false
|
|
10515
10991
|
});
|
|
10516
|
-
const relation = compileTraversalRelation(projectId, input,
|
|
10992
|
+
const relation = compileTraversalRelation(projectId, input, traversal, ctx.source, ctx.correlatedTraversal, ctx.startPredicate);
|
|
10517
10993
|
const order = compileOrder(identityOrderFields());
|
|
10518
10994
|
return {
|
|
10519
|
-
sql: `SELECT selected
|
|
10995
|
+
sql: `SELECT selected.* ${relation.sql} ${ctx.unordered ? "" : `ORDER BY ${order.sql}`} ${ctx.isolateSortOverflow ? "OFFSET 0" : ""}`,
|
|
10520
10996
|
args: [...relation.args, ...ctx.unordered ? [] : order.args],
|
|
10521
10997
|
totalSql: `SELECT COUNT(*)::bigint AS total ${relation.sql}`,
|
|
10522
10998
|
totalArgs: relation.args,
|
|
@@ -10528,15 +11004,15 @@ function compileTraversal(projectId, inputQuery, linkId, direction, sourceObject
|
|
|
10528
11004
|
}
|
|
10529
11005
|
};
|
|
10530
11006
|
}
|
|
10531
|
-
function compileTraversalRelation(projectId, input,
|
|
10532
|
-
const incoming = direction === "incoming";
|
|
11007
|
+
function compileTraversalRelation(projectId, input, traversal, source, correlated = false, endpointPredicate) {
|
|
11008
|
+
const incoming = traversal.direction === "incoming";
|
|
10533
11009
|
const parent = incoming ? "target" : "source";
|
|
10534
11010
|
const endpoint = incoming ? "source" : "target";
|
|
10535
|
-
const
|
|
11011
|
+
const endpointType = compileEndpointTypes("selected", linkEndpointObjectTypeIds(traversal));
|
|
10536
11012
|
return {
|
|
10537
11013
|
sql: `FROM ${source.objectsTable} AS selected
|
|
10538
11014
|
WHERE selected.project_id = ?
|
|
10539
|
-
${
|
|
11015
|
+
${endpointType.sql}
|
|
10540
11016
|
${endpointPredicate ? `AND (${endpointPredicate})` : ""}
|
|
10541
11017
|
AND EXISTS (
|
|
10542
11018
|
SELECT 1 FROM (${input.sql}) AS input
|
|
@@ -10550,12 +11026,15 @@ function compileTraversalRelation(projectId, input, linkId, direction, sourceObj
|
|
|
10550
11026
|
AND edge.link_id = ?
|
|
10551
11027
|
${correlated ? "OFFSET 0" : ""}
|
|
10552
11028
|
)`,
|
|
10553
|
-
args: [
|
|
10554
|
-
|
|
10555
|
-
|
|
10556
|
-
|
|
10557
|
-
|
|
10558
|
-
]
|
|
11029
|
+
args: [projectId, ...endpointType.args, ...input.args, traversal.linkId]
|
|
11030
|
+
};
|
|
11031
|
+
}
|
|
11032
|
+
function compileEndpointTypes(alias, objectTypeIds) {
|
|
11033
|
+
if (!objectTypeIds?.length)
|
|
11034
|
+
return { sql: "", args: [] };
|
|
11035
|
+
return objectTypeIds.length === 1 ? { sql: `AND ${alias}.object_type_id = ?`, args: [objectTypeIds[0]] } : {
|
|
11036
|
+
sql: `AND ${alias}.object_type_id IN (${objectTypeIds.map(() => "?").join(", ")})`,
|
|
11037
|
+
args: [...objectTypeIds]
|
|
10559
11038
|
};
|
|
10560
11039
|
}
|
|
10561
11040
|
function boundedTraversal(query, bounded = false) {
|
|
@@ -10607,7 +11086,7 @@ function compileExpand(projectId, inputQuery, expansions, ctx) {
|
|
|
10607
11086
|
}
|
|
10608
11087
|
function expandRows(input, expansions, ctx) {
|
|
10609
11088
|
const expand = compileExpansionsObject(expansions, { project: "input.project_id", type: "input.object_type_id", id: "input.primary_id" }, "", ctx);
|
|
10610
|
-
const inputOrder = compileOrder(input.order.fields, "input",
|
|
11089
|
+
const inputOrder = compileOrder(input.order.fields, "input", `input.${input.order.propertyColumn}`);
|
|
10611
11090
|
return {
|
|
10612
11091
|
sql: `
|
|
10613
11092
|
SELECT input.*, ${expand.sql} AS _expand
|
|
@@ -10645,7 +11124,8 @@ function compileExpansionValue(expansion, parent, path, ctx) {
|
|
|
10645
11124
|
`${parentId} = ${parent.id}`,
|
|
10646
11125
|
`${edge}.link_id = ?::text`
|
|
10647
11126
|
];
|
|
10648
|
-
const
|
|
11127
|
+
const endpointType = compileEndpointTypes(target, expansion.endpointObjectTypeIds);
|
|
11128
|
+
const whereArgs = [...endpointType.args, expansion.linkId];
|
|
10649
11129
|
if (incoming && expansion.sourceObjectTypeId !== undefined) {
|
|
10650
11130
|
whereParts.push(`${edge}.source_type_id = ?::text`);
|
|
10651
11131
|
whereArgs.push(expansion.sourceObjectTypeId);
|
|
@@ -10656,6 +11136,7 @@ function compileExpansionValue(expansion, parent, path, ctx) {
|
|
|
10656
11136
|
ON ${target}.project_id = ${edge}.project_id
|
|
10657
11137
|
AND ${target}.object_type_id = ${neighborType}
|
|
10658
11138
|
AND ${target}.primary_id = ${neighborId}
|
|
11139
|
+
${endpointType.sql}
|
|
10659
11140
|
WHERE ${whereParts.join(" AND ")}
|
|
10660
11141
|
`;
|
|
10661
11142
|
if (expansion.cardinality === "one") {
|
|
@@ -10718,7 +11199,7 @@ function compileSet(projectId, op, inputs, ctx) {
|
|
|
10718
11199
|
const order = compileOrder(identityOrderFields());
|
|
10719
11200
|
return {
|
|
10720
11201
|
sql: `
|
|
10721
|
-
SELECT
|
|
11202
|
+
SELECT *
|
|
10722
11203
|
FROM ${ctx.source.objectsTable}
|
|
10723
11204
|
WHERE 1 = 0
|
|
10724
11205
|
ORDER BY ${order.sql}
|
|
@@ -10739,7 +11220,7 @@ function compileSet(projectId, op, inputs, ctx) {
|
|
|
10739
11220
|
const order = compileOrder(identityOrderFields());
|
|
10740
11221
|
const selectedOrder = compileOrder(identityOrderFields(), "selected");
|
|
10741
11222
|
const sql = `
|
|
10742
|
-
SELECT selected
|
|
11223
|
+
SELECT selected.*
|
|
10743
11224
|
FROM (${identities.sql}) AS ids
|
|
10744
11225
|
JOIN ${ctx.source.objectsTable} AS selected
|
|
10745
11226
|
ON selected.project_id = ?
|
|
@@ -10765,7 +11246,8 @@ function compileProject(projectId, inputQuery, properties, ctx) {
|
|
|
10765
11246
|
if (!properties)
|
|
10766
11247
|
return input;
|
|
10767
11248
|
const projection = compileProjectionExpression(properties);
|
|
10768
|
-
const
|
|
11249
|
+
const original = `input.${input.order.propertyColumn}`;
|
|
11250
|
+
const inputOrder = compileOrder(input.order.fields, "input", original);
|
|
10769
11251
|
const outputOrder = compileOrder(input.order.fields, undefined, "_cursor_properties");
|
|
10770
11252
|
return {
|
|
10771
11253
|
sql: `
|
|
@@ -10774,7 +11256,7 @@ function compileProject(projectId, inputQuery, properties, ctx) {
|
|
|
10774
11256
|
input.object_type_id,
|
|
10775
11257
|
input.primary_id,
|
|
10776
11258
|
${projection.sql} AS properties,
|
|
10777
|
-
|
|
11259
|
+
${original} AS _cursor_properties,
|
|
10778
11260
|
input.created_at,
|
|
10779
11261
|
input.updated_at,
|
|
10780
11262
|
input.version,
|
|
@@ -10807,7 +11289,7 @@ function compileAggregateSource(projectId, query, source) {
|
|
|
10807
11289
|
return compileAggregateWhere(projectId, query.input, compileTextSearchPredicate(query.query, query.fields, query.fieldsByObjectType), source);
|
|
10808
11290
|
}
|
|
10809
11291
|
case "traverse":
|
|
10810
|
-
return compileAggregateTraversal(projectId, query
|
|
11292
|
+
return compileAggregateTraversal(projectId, query, source);
|
|
10811
11293
|
case "set":
|
|
10812
11294
|
return compileAggregateSet(projectId, query.op, query.inputs, source);
|
|
10813
11295
|
case "sort":
|
|
@@ -10872,8 +11354,11 @@ function compileAggregateWhere(projectId, inputQuery, predicate, source) {
|
|
|
10872
11354
|
args: [...input.args, ...predicate.args]
|
|
10873
11355
|
};
|
|
10874
11356
|
}
|
|
10875
|
-
function compileAggregateTraversal(projectId,
|
|
10876
|
-
const
|
|
11357
|
+
function compileAggregateTraversal(projectId, traversal, source) {
|
|
11358
|
+
const { linkId, direction } = traversal;
|
|
11359
|
+
const input = compileAggregateSource(projectId, traversal.input, source);
|
|
11360
|
+
const endpoints = linkEndpointObjectTypeIds(traversal);
|
|
11361
|
+
const sourceObjectTypeId = endpoints?.length === 1 ? endpoints[0] : undefined;
|
|
10877
11362
|
if (direction === "incoming" && sourceObjectTypeId !== undefined) {
|
|
10878
11363
|
return {
|
|
10879
11364
|
sql: `SELECT selected.project_id, selected.object_type_id, selected.primary_id, selected.properties
|
|
@@ -10894,7 +11379,7 @@ function compileAggregateTraversal(projectId, inputQuery, linkId, direction, sou
|
|
|
10894
11379
|
args: [...input.args, linkId, sourceObjectTypeId, projectId, sourceObjectTypeId]
|
|
10895
11380
|
};
|
|
10896
11381
|
}
|
|
10897
|
-
const relation = compileTraversalRelation(projectId, input,
|
|
11382
|
+
const relation = compileTraversalRelation(projectId, input, traversal, source);
|
|
10898
11383
|
return {
|
|
10899
11384
|
sql: `SELECT selected.project_id, selected.object_type_id, selected.primary_id, selected.properties ${relation.sql}`,
|
|
10900
11385
|
args: relation.args
|
|
@@ -10999,7 +11484,7 @@ function compilePredicate(predicate) {
|
|
|
10999
11484
|
case "constant":
|
|
11000
11485
|
return { sql: predicate.value ? "true" : "false", args: [] };
|
|
11001
11486
|
case "eq":
|
|
11002
|
-
return predicate.scalarKind === "decimal" ? compileDecimalEqualityPredicate(predicate.propertyId, "eq", predicate.value) : compileEqualityPredicate(predicate.propertyId, "eq", predicate.value);
|
|
11487
|
+
return predicate.scalarKind === "decimal" ? compileDecimalEqualityPredicate(predicate.propertyId, "eq", predicate.value) : compileEqualityPredicate(predicate.propertyId, "eq", predicate.value, predicate.scalarKind);
|
|
11003
11488
|
case "neq":
|
|
11004
11489
|
return predicate.scalarKind === "decimal" ? compileDecimalEqualityPredicate(predicate.propertyId, "neq", predicate.value) : compileEqualityPredicate(predicate.propertyId, "neq", predicate.value);
|
|
11005
11490
|
case "lt":
|
|
@@ -11025,7 +11510,7 @@ function compilePredicate(predicate) {
|
|
|
11025
11510
|
};
|
|
11026
11511
|
}
|
|
11027
11512
|
case "in":
|
|
11028
|
-
return predicate.scalarKind === "decimal" ? compileDecimalInPredicate(predicate.propertyId, predicate.values) : compileInPredicate(predicate.propertyId, predicate.values);
|
|
11513
|
+
return predicate.scalarKind === "decimal" ? compileDecimalInPredicate(predicate.propertyId, predicate.values) : compileInPredicate(predicate.propertyId, predicate.values, predicate.scalarKind);
|
|
11029
11514
|
case "exists": {
|
|
11030
11515
|
const sql = `jsonb_exists(properties, ?::text)`;
|
|
11031
11516
|
const item = { sql, args: [predicate.propertyId] };
|
|
@@ -11077,13 +11562,13 @@ function negatePredicate(predicate) {
|
|
|
11077
11562
|
args: predicate.args
|
|
11078
11563
|
};
|
|
11079
11564
|
}
|
|
11080
|
-
function compileEqualityPredicate(propertyId, op, value) {
|
|
11565
|
+
function compileEqualityPredicate(propertyId, op, value, scalarKind) {
|
|
11081
11566
|
if (value === null) {
|
|
11082
11567
|
const sql = `${jsonTypeExpression()} = 'null'`;
|
|
11083
11568
|
const item = { sql, args: [propertyId] };
|
|
11084
11569
|
return op === "eq" ? item : negatePredicate(item);
|
|
11085
11570
|
}
|
|
11086
|
-
if (typeof value === "string") {
|
|
11571
|
+
if (typeof value === "string" && !(op === "eq" && hasJsonValueIndex(scalarKind))) {
|
|
11087
11572
|
if (op === "eq") {
|
|
11088
11573
|
return {
|
|
11089
11574
|
sql: `(${jsonTypeExpression()} = 'string' AND ${jsonTextExpression()} = ?::text AND md5(${jsonTextExpression()}) = md5(?::text))`,
|
|
@@ -11106,12 +11591,13 @@ function compileEqualityPredicate(propertyId, op, value) {
|
|
|
11106
11591
|
args: [propertyId, propertyId, propertyId, jsonbValue(value)]
|
|
11107
11592
|
};
|
|
11108
11593
|
}
|
|
11109
|
-
function compileInPredicate(propertyId, values) {
|
|
11594
|
+
function compileInPredicate(propertyId, values, scalarKind) {
|
|
11110
11595
|
if (values.length === 0)
|
|
11111
11596
|
return { sql: "0 = 1", args: [] };
|
|
11112
11597
|
const nonNullValues = values.filter((value) => value !== null);
|
|
11113
|
-
const
|
|
11114
|
-
const
|
|
11598
|
+
const comparedAsText = (value) => typeof value === "string" && !hasJsonValueIndex(scalarKind);
|
|
11599
|
+
const stringValues = nonNullValues.filter(comparedAsText);
|
|
11600
|
+
const jsonbValues = nonNullValues.filter((value) => !comparedAsText(value));
|
|
11115
11601
|
const clauses = [];
|
|
11116
11602
|
const args = [];
|
|
11117
11603
|
if (values.length !== nonNullValues.length) {
|
|
@@ -11401,6 +11887,9 @@ function orderedValueExpression(scalarKind, propertyColumn = "properties") {
|
|
|
11401
11887
|
return `((${jsonTextExpression(propertyColumn)}) COLLATE "C")`;
|
|
11402
11888
|
return `NULLIF(${jsonValueExpression(propertyColumn)}, 'null'::jsonb)`;
|
|
11403
11889
|
}
|
|
11890
|
+
function hasJsonValueIndex(scalarKind) {
|
|
11891
|
+
return scalarKind === "date" || scalarKind === "timestamp" || scalarKind === "uuid";
|
|
11892
|
+
}
|
|
11404
11893
|
function isTextScalarKind(scalarKind) {
|
|
11405
11894
|
return scalarKind === "string" || scalarKind === "uuid";
|
|
11406
11895
|
}
|
|
@@ -11467,7 +11956,7 @@ function isPlainObject(value) {
|
|
|
11467
11956
|
// src/objects/query-indexes.ts
|
|
11468
11957
|
async function ensurePgObjectQueryIndexes(sql, schema, indexes) {
|
|
11469
11958
|
const definitions = indexes.map((index) => ({
|
|
11470
|
-
table: index.kind === "incomingLinks" ? "links" :
|
|
11959
|
+
table: index.kind === "incomingLinks" ? "links" : objectPartitionName(index.objectTypeId),
|
|
11471
11960
|
expression: compileIndex(index),
|
|
11472
11961
|
index
|
|
11473
11962
|
}));
|
|
@@ -11486,6 +11975,9 @@ async function ensurePgObjectQueryIndexes(sql, schema, indexes) {
|
|
|
11486
11975
|
let connected = true;
|
|
11487
11976
|
try {
|
|
11488
11977
|
await lockPgPreparation(connection, `${schema}.objects.query-indexes`);
|
|
11978
|
+
for (const objectTypeId of new Set(indexes.flatMap((index) => index.kind === "incomingLinks" ? [] : [index.objectTypeId]))) {
|
|
11979
|
+
await ensureObjectPartition(connection, schema, objectTypeId);
|
|
11980
|
+
}
|
|
11489
11981
|
for (const definition of definitions) {
|
|
11490
11982
|
names.push(await createQueryIndex(connection, schema, definition));
|
|
11491
11983
|
}
|
|
@@ -11501,8 +11993,9 @@ async function ensurePgObjectQueryIndexes(sql, schema, indexes) {
|
|
|
11501
11993
|
return names;
|
|
11502
11994
|
}
|
|
11503
11995
|
async function createQueryIndex(connection, schema, definition) {
|
|
11504
|
-
const
|
|
11505
|
-
|
|
11996
|
+
const { table } = definition;
|
|
11997
|
+
const name = `sixb_query_${createHash4("sha256").update(`${table}:${definition.expression}`).digest("hex").slice(0, 32)}`;
|
|
11998
|
+
await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${identifier(schema)}.${identifier(table)} ${definition.expression}`);
|
|
11506
11999
|
const [index] = await connection`
|
|
11507
12000
|
SELECT i.indisvalid AS valid FROM pg_index i
|
|
11508
12001
|
JOIN pg_class c ON c.oid = i.indexrelid
|
|
@@ -11512,8 +12005,7 @@ async function createQueryIndex(connection, schema, definition) {
|
|
|
11512
12005
|
throw new Error(`[SixbPg] Query index ${schema}.${name} is invalid after an interrupted build. Drop that index concurrently and retry ensureObjectQueryIndexes().`);
|
|
11513
12006
|
if (definition.index.kind === "sort") {
|
|
11514
12007
|
await connection.unsafe(`COMMENT ON INDEX ${identifier(schema)}.${identifier(name)} IS ${literal(JSON.stringify(definition.index))}`);
|
|
11515
|
-
|
|
11516
|
-
await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(`${name}_overflow`)} ON ${identifier(schema)}.objects (primary_id) WHERE ${scope} AND NOT (${sortIndexBound(definition.index)})`);
|
|
12008
|
+
await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(`${name}_overflow`)} ON ${identifier(schema)}.${identifier(table)} (primary_id) WHERE project_id = ${literal(definition.index.projectId)} AND NOT (${sortIndexBound(definition.index)})`);
|
|
11517
12009
|
const superseded = await connection`
|
|
11518
12010
|
SELECT c.relname AS name FROM pg_index i
|
|
11519
12011
|
JOIN pg_class c ON c.oid = i.indexrelid
|
|
@@ -11529,24 +12021,33 @@ async function createQueryIndex(connection, schema, definition) {
|
|
|
11529
12021
|
return name;
|
|
11530
12022
|
}
|
|
11531
12023
|
async function ensureExpressionStatistics(connection, schema, indexes) {
|
|
11532
|
-
|
|
11533
|
-
const
|
|
11534
|
-
|
|
11535
|
-
const name = `sixb_query_stats_${createHash3("sha256").update(expression).digest("hex").slice(0, 24)}`;
|
|
11536
|
-
const [existing] = await connection`
|
|
11537
|
-
SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
|
|
11538
|
-
WHERE n.nspname = ${schema} AND s.stxname = ${name}`;
|
|
11539
|
-
if (existing)
|
|
12024
|
+
const expressionsByType = new Map;
|
|
12025
|
+
for (const index of indexes) {
|
|
12026
|
+
if (index.kind === "incomingLinks")
|
|
11540
12027
|
continue;
|
|
11541
|
-
|
|
11542
|
-
|
|
12028
|
+
const expressions = expressionsByType.get(index.objectTypeId) ?? new Set;
|
|
12029
|
+
for (const expression of statisticsExpressions(index))
|
|
12030
|
+
expressions.add(expression);
|
|
12031
|
+
expressionsByType.set(index.objectTypeId, expressions);
|
|
12032
|
+
}
|
|
12033
|
+
for (const [objectTypeId, expressions] of expressionsByType) {
|
|
12034
|
+
const partition = `${identifier(schema)}.${identifier(objectPartitionName(objectTypeId))}`;
|
|
12035
|
+
let statisticsChanged = false;
|
|
12036
|
+
for (const expression of expressions) {
|
|
12037
|
+
const name = `sixb_query_stats_${createHash4("sha256").update(`${objectTypeId}:${expression}`).digest("hex").slice(0, 24)}`;
|
|
12038
|
+
const [existing] = await connection`
|
|
12039
|
+
SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
|
|
12040
|
+
WHERE n.nspname = ${schema} AND s.stxname = ${name}`;
|
|
12041
|
+
if (existing)
|
|
12042
|
+
continue;
|
|
12043
|
+
await connection.unsafe(`CREATE STATISTICS ${identifier(schema)}.${identifier(name)} ON project_id, ${expression} FROM ${partition}`);
|
|
12044
|
+
statisticsChanged = true;
|
|
12045
|
+
}
|
|
12046
|
+
if (statisticsChanged)
|
|
12047
|
+
await connection.unsafe(`ANALYZE ${partition}`);
|
|
11543
12048
|
}
|
|
11544
|
-
if (statisticsChanged)
|
|
11545
|
-
await connection.unsafe(`ANALYZE ${identifier(schema)}.objects`);
|
|
11546
12049
|
}
|
|
11547
12050
|
function statisticsExpressions(index) {
|
|
11548
|
-
if (index.kind === "incomingLinks")
|
|
11549
|
-
return [];
|
|
11550
12051
|
const equality = new Set(Object.keys(index.where ?? {}));
|
|
11551
12052
|
if (index.kind === "filter")
|
|
11552
12053
|
for (const property of index.properties)
|
|
@@ -11570,10 +12071,7 @@ function compileIndex(index) {
|
|
|
11570
12071
|
throw new Error("[SixbPg] Incoming link indexes do not accept object property conditions");
|
|
11571
12072
|
return `(target_id, link_id, source_type_id, source_id) INCLUDE (project_id, target_type_id) WHERE project_id = ${literal(index.projectId)} AND target_type_id = ${literal(index.objectTypeId)}`;
|
|
11572
12073
|
}
|
|
11573
|
-
const predicate = [
|
|
11574
|
-
`project_id = ${literal(index.projectId)}`,
|
|
11575
|
-
`object_type_id = ${literal(index.objectTypeId)}`
|
|
11576
|
-
];
|
|
12074
|
+
const predicate = [`project_id = ${literal(index.projectId)}`];
|
|
11577
12075
|
for (const [propertyId, value] of Object.entries(index.where ?? {}).sort(([a], [b]) => a.localeCompare(b))) {
|
|
11578
12076
|
const property = `properties -> ${literal(propertyId)}`;
|
|
11579
12077
|
if (value === null)
|
|
@@ -11626,14 +12124,6 @@ function sortValue(field) {
|
|
|
11626
12124
|
return `(${text} COLLATE "C")`;
|
|
11627
12125
|
return `(NULLIF(properties -> ${literal(field.propertyId)}, 'null'::jsonb))`;
|
|
11628
12126
|
}
|
|
11629
|
-
function literal(value) {
|
|
11630
|
-
if (value.includes("\x00"))
|
|
11631
|
-
throw new Error("[SixbPg] Query index names and values cannot contain NUL");
|
|
11632
|
-
return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`;
|
|
11633
|
-
}
|
|
11634
|
-
function identifier(value) {
|
|
11635
|
-
return `"${value.replace(/"/g, '""')}"`;
|
|
11636
|
-
}
|
|
11637
12127
|
function sortIndexBound(index) {
|
|
11638
12128
|
const sizes = [
|
|
11639
12129
|
"octet_length(primary_id)",
|
|
@@ -11669,8 +12159,8 @@ async function readPgSortIndexes(sql) {
|
|
|
11669
12159
|
JOIN pg_class c ON c.oid = i.indexrelid
|
|
11670
12160
|
JOIN pg_description d
|
|
11671
12161
|
ON d.objoid = c.oid AND d.classoid = 'pg_class'::regclass AND d.objsubid = 0
|
|
11672
|
-
WHERE i.indrelid = 'objects'::regclass
|
|
11673
|
-
AND c.relname LIKE 'sixb_query_%'
|
|
12162
|
+
WHERE i.indrelid IN (SELECT inhrelid FROM pg_inherits WHERE inhparent = 'objects'::regclass)
|
|
12163
|
+
AND i.indisvalid AND c.relname LIKE 'sixb_query_%'
|
|
11674
12164
|
`;
|
|
11675
12165
|
const indexes = [];
|
|
11676
12166
|
for (const row of rows) {
|
|
@@ -11705,9 +12195,9 @@ async function findPgSortIndexBound(sql, catalog, projectId, query) {
|
|
|
11705
12195
|
const inspect = (predicate) => {
|
|
11706
12196
|
if (predicate.op === "and")
|
|
11707
12197
|
predicate.items.forEach(inspect);
|
|
11708
|
-
else if (predicate.op === "eq" && typeof predicate.value === "string")
|
|
12198
|
+
else if (predicate.op === "eq" && typeof predicate.value === "string" && !hasJsonValueIndex(predicate.scalarKind))
|
|
11709
12199
|
equality.add(predicate.propertyId);
|
|
11710
|
-
else if (predicate.op === "in" && predicate.values.length === 1 && typeof predicate.values[0] === "string")
|
|
12200
|
+
else if (predicate.op === "in" && predicate.values.length === 1 && typeof predicate.values[0] === "string" && !hasJsonValueIndex(predicate.scalarKind))
|
|
11711
12201
|
equality.add(predicate.propertyId);
|
|
11712
12202
|
};
|
|
11713
12203
|
inspect(input.predicate);
|
|
@@ -11717,8 +12207,9 @@ async function findPgSortIndexBound(sql, catalog, projectId, query) {
|
|
|
11717
12207
|
let objectTypeId;
|
|
11718
12208
|
if (input.kind === "start" && !input.includeSubtypes) {
|
|
11719
12209
|
objectTypeId = input.objectTypeId;
|
|
11720
|
-
} else if (input.kind === "traverse"
|
|
11721
|
-
|
|
12210
|
+
} else if (input.kind === "traverse") {
|
|
12211
|
+
const endpoints = linkEndpointObjectTypeIds2(input);
|
|
12212
|
+
objectTypeId = endpoints?.length === 1 ? endpoints[0] : undefined;
|
|
11722
12213
|
}
|
|
11723
12214
|
if (!objectTypeId || !fields?.length || fields.some((field) => field.kind !== "property"))
|
|
11724
12215
|
return;
|
|
@@ -12221,7 +12712,7 @@ function linkIdentity(link) {
|
|
|
12221
12712
|
}
|
|
12222
12713
|
|
|
12223
12714
|
// src/objects/text-count-index.ts
|
|
12224
|
-
import { createHash as
|
|
12715
|
+
import { createHash as createHash5 } from "crypto";
|
|
12225
12716
|
function describe(input) {
|
|
12226
12717
|
const definition = { ...input, filters: [...new Set(input.filters ?? [])].sort() };
|
|
12227
12718
|
if (definition.filters.length > 4)
|
|
@@ -12232,7 +12723,7 @@ function describe(input) {
|
|
|
12232
12723
|
input.propertyId,
|
|
12233
12724
|
...definition.filters
|
|
12234
12725
|
])
|
|
12235
|
-
|
|
12726
|
+
literal(value);
|
|
12236
12727
|
const key = hash(JSON.stringify([
|
|
12237
12728
|
definition.projectId,
|
|
12238
12729
|
definition.objectTypeId,
|
|
@@ -12244,10 +12735,10 @@ function describe(input) {
|
|
|
12244
12735
|
property,
|
|
12245
12736
|
`sixb_qfilter_${hash(JSON.stringify([input.projectId, input.objectTypeId, property]))}`
|
|
12246
12737
|
]));
|
|
12247
|
-
const scope = `project_id = ${
|
|
12738
|
+
const scope = `project_id = ${literal(input.projectId)} AND object_type_id = ${literal(input.objectTypeId)}`;
|
|
12248
12739
|
const compact = [
|
|
12249
|
-
`octet_length(${
|
|
12250
|
-
...[...filters.values()].map((column) => `(${
|
|
12740
|
+
`octet_length(${identifier(textColumn)}) <= 1024`,
|
|
12741
|
+
...[...filters.values()].map((column) => `(${identifier(column)} IS NULL OR octet_length(${identifier(column)}) <= 128)`)
|
|
12251
12742
|
].join(" AND ");
|
|
12252
12743
|
return { definition, name: `sixb_text_count_${key}`, textColumn, filters, scope, compact };
|
|
12253
12744
|
}
|
|
@@ -12263,7 +12754,7 @@ async function preparePgObjectTextCounts(sql, schema, inputs) {
|
|
|
12263
12754
|
`;
|
|
12264
12755
|
if (!extension?.ready)
|
|
12265
12756
|
throw new Error("[SixbPg] Text counts require CREATE EXTENSION pg_trgm WITH SCHEMA public");
|
|
12266
|
-
const table = `${
|
|
12757
|
+
const table = `${identifier(schema)}.objects`;
|
|
12267
12758
|
const names = [];
|
|
12268
12759
|
await withReservedPgConnection(sql, async (connection) => {
|
|
12269
12760
|
let connected = true;
|
|
@@ -12276,29 +12767,36 @@ async function preparePgObjectTextCounts(sql, schema, inputs) {
|
|
|
12276
12767
|
const columns = new Set(existing.map((row) => row.attname));
|
|
12277
12768
|
const additions = new Map;
|
|
12278
12769
|
for (const item of definitions) {
|
|
12279
|
-
additions.set(item.textColumn, `CASE WHEN ${item.scope} THEN lower(coalesce(properties ->> ${
|
|
12770
|
+
additions.set(item.textColumn, `CASE WHEN ${item.scope} THEN lower(coalesce(properties ->> ${literal(item.definition.propertyId)}, '')) END`);
|
|
12280
12771
|
for (const [property, column] of item.filters)
|
|
12281
|
-
additions.set(column, `CASE WHEN ${item.scope} AND jsonb_typeof(properties -> ${
|
|
12772
|
+
additions.set(column, `CASE WHEN ${item.scope} AND jsonb_typeof(properties -> ${literal(property)}) = 'string' THEN properties ->> ${literal(property)} END`);
|
|
12282
12773
|
}
|
|
12283
12774
|
const missing = [...additions].filter(([column]) => !columns.has(column));
|
|
12284
12775
|
if (missing.length > 0) {
|
|
12285
|
-
await connection.unsafe(`ALTER TABLE ${table} ${missing.map(([column, expression]) => `ADD COLUMN ${
|
|
12776
|
+
await connection.unsafe(`ALTER TABLE ${table} ${missing.map(([column, expression]) => `ADD COLUMN ${identifier(column)} text GENERATED ALWAYS AS (${expression}) STORED`).join(", ")}`);
|
|
12777
|
+
}
|
|
12778
|
+
const objectTypeIds = new Set(definitions.map((item) => item.definition.objectTypeId));
|
|
12779
|
+
for (const objectTypeId of objectTypeIds) {
|
|
12780
|
+
await ensureObjectPartition(connection, schema, objectTypeId);
|
|
12286
12781
|
}
|
|
12287
12782
|
for (const item of definitions) {
|
|
12288
12783
|
const statements = textCountIndexStatements(item);
|
|
12784
|
+
const partition = `${identifier(schema)}.${identifier(objectPartitionName(item.definition.objectTypeId))}`;
|
|
12289
12785
|
for (const [name, expression] of statements) {
|
|
12290
|
-
await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${
|
|
12786
|
+
await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${partition} ${expression}`);
|
|
12291
12787
|
const [valid] = await connection`
|
|
12292
12788
|
SELECT indisvalid FROM pg_index
|
|
12293
|
-
WHERE indexrelid = ${`${
|
|
12789
|
+
WHERE indexrelid = ${`${identifier(schema)}.${identifier(name)}`}::regclass
|
|
12294
12790
|
`;
|
|
12295
12791
|
if (!valid?.indisvalid)
|
|
12296
12792
|
throw new Error(`[SixbPg] Text count index ${schema}.${name} is invalid. Drop it concurrently and retry preparation.`);
|
|
12297
12793
|
}
|
|
12298
|
-
await connection.unsafe(`COMMENT ON INDEX ${
|
|
12794
|
+
await connection.unsafe(`COMMENT ON INDEX ${identifier(schema)}.${identifier(item.name)} IS ${literal(JSON.stringify(item.definition))}`);
|
|
12299
12795
|
names.push(item.name);
|
|
12300
12796
|
}
|
|
12301
|
-
|
|
12797
|
+
for (const objectTypeId of objectTypeIds) {
|
|
12798
|
+
await connection.unsafe(`ANALYZE ${identifier(schema)}.${identifier(objectPartitionName(objectTypeId))}`);
|
|
12799
|
+
}
|
|
12302
12800
|
} catch (error) {
|
|
12303
12801
|
connected = !isConnectionLost(error);
|
|
12304
12802
|
throw error;
|
|
@@ -12314,16 +12812,16 @@ function textCountIndexStatements(item) {
|
|
|
12314
12812
|
const statements = [
|
|
12315
12813
|
[
|
|
12316
12814
|
item.name,
|
|
12317
|
-
`(${[...item.filters.values(), item.textColumn].map(
|
|
12815
|
+
`(${[...item.filters.values(), item.textColumn].map(identifier).join(", ")}) WHERE ${predicate}`
|
|
12318
12816
|
],
|
|
12319
12817
|
[
|
|
12320
12818
|
`${item.name}_gin`,
|
|
12321
|
-
`USING gin (${
|
|
12819
|
+
`USING gin (${identifier(item.textColumn)} public.gin_trgm_ops) WHERE ${predicate}`
|
|
12322
12820
|
],
|
|
12323
12821
|
[`${item.name}_overflow`, `(primary_id) WHERE ${item.scope} AND NOT (${item.compact})`]
|
|
12324
12822
|
];
|
|
12325
12823
|
if (item.filters.size > 0) {
|
|
12326
|
-
const columns = [...item.filters.values()].map(
|
|
12824
|
+
const columns = [...item.filters.values()].map(identifier);
|
|
12327
12825
|
const compact = columns.map((column) => `(${column} IS NOT NULL AND octet_length(${column}) <= 128)`).join(" AND ");
|
|
12328
12826
|
statements.push([`${item.name}_facets`, `(${columns.join(", ")}) WHERE ${item.scope} AND (${compact})`], [`${item.name}_facet_overflow`, `(primary_id) WHERE ${item.scope} AND NOT (${compact})`], [
|
|
12329
12827
|
`${item.name}_text_facets_overflow`,
|
|
@@ -12345,7 +12843,7 @@ async function compilePgIndexedTextAggregate(sql, projectId, query, facets) {
|
|
|
12345
12843
|
const rows = await sql`
|
|
12346
12844
|
SELECT c.relname AS name, obj_description(c.oid,'pg_class') AS definition
|
|
12347
12845
|
FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid
|
|
12348
|
-
WHERE i.indrelid=
|
|
12846
|
+
WHERE i.indrelid=to_regclass(${objectPartitionName(shape.objectTypeId)}) AND i.indisvalid
|
|
12349
12847
|
AND c.relname LIKE 'sixb_text_count_%' AND obj_description(c.oid,'pg_class') IS NOT NULL`;
|
|
12350
12848
|
for (const row of rows) {
|
|
12351
12849
|
const definition = parseDefinition(row.definition);
|
|
@@ -12390,8 +12888,8 @@ async function compilePgIndexedTextAggregate(sql, projectId, query, facets) {
|
|
|
12390
12888
|
broadWhere.push("false");
|
|
12391
12889
|
}
|
|
12392
12890
|
for (const term of terms) {
|
|
12393
|
-
where.push(`${
|
|
12394
|
-
broadWhere.push(`position(${bind(term)}::text in ${
|
|
12891
|
+
where.push(`${identifier(item.textColumn)} LIKE ${bind(`%${term.replace(/[\\%_]/g, "\\$&")}%`)}::text`);
|
|
12892
|
+
broadWhere.push(`position(${bind(term)}::text in ${identifier(item.textColumn)}) > 0`);
|
|
12395
12893
|
}
|
|
12396
12894
|
}
|
|
12397
12895
|
const input = { projectId, query, item, where, broadWhere, args };
|
|
@@ -12402,12 +12900,12 @@ async function compilePgIndexedTextAggregate(sql, projectId, query, facets) {
|
|
|
12402
12900
|
return;
|
|
12403
12901
|
}
|
|
12404
12902
|
function compileIndexedFacets({ projectId, query, item, where, broadWhere, args }, facets, hasText, bind) {
|
|
12405
|
-
const allString = [...item.filters.values()].map((column) => `${
|
|
12406
|
-
const filterCompact = [...item.filters.values()].map((column) => `(${
|
|
12903
|
+
const allString = [...item.filters.values()].map((column) => `${identifier(column)} IS NOT NULL`).join(" AND ") || "true";
|
|
12904
|
+
const filterCompact = [...item.filters.values()].map((column) => `(${identifier(column)} IS NOT NULL AND octet_length(${identifier(column)}) <= 128)`).join(" AND ") || "true";
|
|
12407
12905
|
const compact = hasText ? `(${item.compact}) AND (${allString})` : filterCompact;
|
|
12408
12906
|
const nativeWhere = hasText ? [...where, allString] : [where[0], where[1], compact, ...where.slice(3)];
|
|
12409
12907
|
const nativeBroad = [...broadWhere, allString];
|
|
12410
|
-
const fields = facets.map((facet, i) => `'string'::text AS f${i}_type, ${
|
|
12908
|
+
const fields = facets.map((facet, i) => `'string'::text AS f${i}_type, ${identifier(item.filters.get(facet.propertyId))} AS f${i}_value`).join(", ");
|
|
12411
12909
|
const overflow = compilePgObjectQuery(projectId, query, {
|
|
12412
12910
|
includeTotal: false,
|
|
12413
12911
|
source: {
|
|
@@ -12485,7 +12983,7 @@ function stringPredicate(predicate, columns, bind) {
|
|
|
12485
12983
|
const values = predicate.op === "in" ? predicate.values : [predicate.value];
|
|
12486
12984
|
if (!values.every((value) => typeof value === "string"))
|
|
12487
12985
|
return;
|
|
12488
|
-
return values.length ? `${
|
|
12986
|
+
return values.length ? `${identifier(column)} IN (${values.map((value) => `${bind(value)}::text`).join(", ")})` : "false";
|
|
12489
12987
|
}
|
|
12490
12988
|
function parseDefinition(text) {
|
|
12491
12989
|
try {
|
|
@@ -12503,15 +13001,7 @@ function parseDefinition(text) {
|
|
|
12503
13001
|
}
|
|
12504
13002
|
}
|
|
12505
13003
|
function hash(value) {
|
|
12506
|
-
return
|
|
12507
|
-
}
|
|
12508
|
-
function identifier2(value) {
|
|
12509
|
-
return `"${value.replace(/"/g, '""')}"`;
|
|
12510
|
-
}
|
|
12511
|
-
function literal2(value) {
|
|
12512
|
-
if (value.includes("\x00"))
|
|
12513
|
-
throw new Error("[SixbPg] Text count identifiers cannot contain NUL");
|
|
12514
|
-
return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`;
|
|
13004
|
+
return createHash5("sha256").update(value).digest("hex").slice(0, 24);
|
|
12515
13005
|
}
|
|
12516
13006
|
|
|
12517
13007
|
// src/objects/reader.ts
|
|
@@ -13326,7 +13816,7 @@ function assertReconciliationPageLimit(limit) {
|
|
|
13326
13816
|
}
|
|
13327
13817
|
}
|
|
13328
13818
|
// src/objects/prepare-queries.ts
|
|
13329
|
-
import { createHash as
|
|
13819
|
+
import { createHash as createHash6 } from "crypto";
|
|
13330
13820
|
async function preparePgObjectQueries(sql, schema, plan) {
|
|
13331
13821
|
const fingerprint = preparationFingerprint(plan);
|
|
13332
13822
|
const current = await readCompletedPreparation(sql, fingerprint);
|
|
@@ -13351,17 +13841,18 @@ async function preparePgObjectQueries(sql, schema, plan) {
|
|
|
13351
13841
|
return result;
|
|
13352
13842
|
}, unlock);
|
|
13353
13843
|
await unlock();
|
|
13354
|
-
return result;
|
|
13844
|
+
return result.status === "current" ? result : { ...result, warnings: [...result.warnings, ...await linkWarnings(connection, plan)] };
|
|
13355
13845
|
});
|
|
13356
13846
|
}
|
|
13357
13847
|
function preparationFingerprint(plan) {
|
|
13358
13848
|
const objectTypes = plan.objectTypes.map((type) => ({
|
|
13359
13849
|
objectTypeId: type.objectTypeId,
|
|
13360
13850
|
properties: [...type.properties].sort((a, b) => a.propertyId.localeCompare(b.propertyId)),
|
|
13361
|
-
indexes: type.indexes.map(normalizeIndex).sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)))
|
|
13851
|
+
indexes: type.indexes.map(normalizeIndex).sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
|
|
13852
|
+
links: [...type.links].sort((a, b) => a.linkId.localeCompare(b.linkId))
|
|
13362
13853
|
})).sort((a, b) => a.objectTypeId.localeCompare(b.objectTypeId));
|
|
13363
|
-
const definition = { version:
|
|
13364
|
-
return
|
|
13854
|
+
const definition = { version: 3, projectId: plan.projectId, objectTypes };
|
|
13855
|
+
return createHash6("sha256").update(JSON.stringify(definition)).digest("hex");
|
|
13365
13856
|
}
|
|
13366
13857
|
function normalizeIndex(index) {
|
|
13367
13858
|
if (index.kind === "sort") {
|
|
@@ -13394,6 +13885,8 @@ async function prepareStructures(sql, schema, plan) {
|
|
|
13394
13885
|
if (indexes.some((index) => index.kind === "text") || textCounts.length > 0) {
|
|
13395
13886
|
await ensureTrigrams(sql);
|
|
13396
13887
|
}
|
|
13888
|
+
for (const type of plan.objectTypes)
|
|
13889
|
+
await ensureObjectPartition(sql, schema, type.objectTypeId);
|
|
13397
13890
|
const names = await ensurePgObjectQueryIndexes(sql, schema, indexes);
|
|
13398
13891
|
const texts = await preparePgObjectTextCounts(sql, schema, textCounts);
|
|
13399
13892
|
return {
|
|
@@ -13403,6 +13896,27 @@ async function prepareStructures(sql, schema, plan) {
|
|
|
13403
13896
|
warnings
|
|
13404
13897
|
};
|
|
13405
13898
|
}
|
|
13899
|
+
async function linkWarnings(sql, plan) {
|
|
13900
|
+
const declared = new Map(plan.objectTypes.flatMap((type) => type.links.map((link) => [`${type.objectTypeId}\x00${link.linkId}`, link])));
|
|
13901
|
+
try {
|
|
13902
|
+
const edges = await sql`
|
|
13903
|
+
SELECT source_type_id, link_id, target_type_id, COUNT(*) AS edges
|
|
13904
|
+
FROM links WHERE project_id = ${plan.projectId}
|
|
13905
|
+
GROUP BY source_type_id, link_id, target_type_id`;
|
|
13906
|
+
return edges.flatMap((edge) => {
|
|
13907
|
+
const name = `${edge.source_type_id}.${edge.link_id}`;
|
|
13908
|
+
const link = declared.get(`${edge.source_type_id}\x00${edge.link_id}`);
|
|
13909
|
+
if (!link) {
|
|
13910
|
+
return `${name}: ${edge.edges} stored edges to ${edge.target_type_id} belong to a link the ontology no longer declares. Queries may not return them.`;
|
|
13911
|
+
}
|
|
13912
|
+
return !link.targetObjectTypeIds || link.targetObjectTypeIds.includes(edge.target_type_id) ? [] : `${name}: ${edge.edges} stored edges point to ${edge.target_type_id}, which this link no longer allows. Queries may not return them.`;
|
|
13913
|
+
});
|
|
13914
|
+
} catch (error) {
|
|
13915
|
+
if (isConnectionLost(error))
|
|
13916
|
+
throw error;
|
|
13917
|
+
return [`Could not check stored links against the ontology: ${String(error)}`];
|
|
13918
|
+
}
|
|
13919
|
+
}
|
|
13406
13920
|
function compilePreparationPlan(plan) {
|
|
13407
13921
|
const indexes = [];
|
|
13408
13922
|
const textCounts = [];
|
|
@@ -13662,12 +14176,16 @@ class PgOntologyCommitStorage {
|
|
|
13662
14176
|
assertNonnegativeInteger(offset, "Ontology commit list offset");
|
|
13663
14177
|
if (limit !== undefined)
|
|
13664
14178
|
assertNonnegativeInteger(limit, "Ontology commit list limit");
|
|
14179
|
+
if (input.committedAt !== undefined) {
|
|
14180
|
+
assertTimestamp(input.committedAt, "Ontology commit list committedAt");
|
|
14181
|
+
}
|
|
13665
14182
|
const runFilter = input.run?.kind === "action" ? sql`AND origin_kind = 'action' AND origin_run_id = ${input.run.id}` : input.run?.kind === "projection" ? sql`AND origin_run_id = ${input.run.id}
|
|
13666
14183
|
AND origin_kind IN ('projection', 'telemetry')` : sql``;
|
|
14184
|
+
const committedAtFilter = input.committedAt === undefined ? sql`` : sql`AND committed_at = ${input.committedAt}::timestamptz`;
|
|
13667
14185
|
const [{ total }] = await sql`
|
|
13668
14186
|
SELECT COUNT(*) AS total
|
|
13669
14187
|
FROM ontology_commits
|
|
13670
|
-
WHERE project_id = ${input.projectId} ${runFilter}
|
|
14188
|
+
WHERE project_id = ${input.projectId} ${runFilter} ${committedAtFilter}
|
|
13671
14189
|
`;
|
|
13672
14190
|
const direction = input.order === "desc" ? sql`DESC` : sql`ASC`;
|
|
13673
14191
|
const order = input.run?.kind === "projection" ? sql`CASE WHEN origin_kind = 'telemetry' THEN origin_batch_ordinal ELSE -1 END
|
|
@@ -13675,7 +14193,7 @@ class PgOntologyCommitStorage {
|
|
|
13675
14193
|
const limitFragment = limit === undefined ? sql`` : sql`LIMIT ${limit}`;
|
|
13676
14194
|
const rows = await sql`
|
|
13677
14195
|
SELECT * FROM ontology_commits
|
|
13678
|
-
WHERE project_id = ${input.projectId} ${runFilter}
|
|
14196
|
+
WHERE project_id = ${input.projectId} ${runFilter} ${committedAtFilter}
|
|
13679
14197
|
ORDER BY ${order}
|
|
13680
14198
|
${limitFragment} OFFSET ${offset}
|
|
13681
14199
|
`;
|
|
@@ -13711,6 +14229,7 @@ import {
|
|
|
13711
14229
|
beginMaterializationVectorChanges,
|
|
13712
14230
|
cardinalityViolation,
|
|
13713
14231
|
invalidCorrelation as invalidCorrelation2,
|
|
14232
|
+
prepareMaterializationStateRead,
|
|
13714
14233
|
sameNonnegativeCounts as sameCounts,
|
|
13715
14234
|
uniqueSorted
|
|
13716
14235
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
@@ -15415,7 +15934,7 @@ async function sourceRow(sql, projectId, sourceId, materializationId, lock = fal
|
|
|
15415
15934
|
class PgMaterializationSessionState extends ProviderMaterializationSessionState {
|
|
15416
15935
|
plan;
|
|
15417
15936
|
stagedWorkCount = 0;
|
|
15418
|
-
changedObjects =
|
|
15937
|
+
changedObjects = new Map;
|
|
15419
15938
|
changedLinks = 0;
|
|
15420
15939
|
changedUniqueValues = 0;
|
|
15421
15940
|
uniqueWork = { releases: false, claims: false };
|
|
@@ -15463,6 +15982,9 @@ class PgMaterializationSessions {
|
|
|
15463
15982
|
return;
|
|
15464
15983
|
if (!session.plan)
|
|
15465
15984
|
await this.deleteWork(session.id);
|
|
15985
|
+
this.discard(session);
|
|
15986
|
+
}
|
|
15987
|
+
discard(session) {
|
|
15466
15988
|
session.active = false;
|
|
15467
15989
|
this.live.delete(session);
|
|
15468
15990
|
this.context?.materializations.complete(session.providerToken);
|
|
@@ -15824,6 +16346,18 @@ class PgMaterializationWriter {
|
|
|
15824
16346
|
await this.writeTouches(target);
|
|
15825
16347
|
return { objectWrites, linkWrites, uniqueValueWrites, eventCount };
|
|
15826
16348
|
}
|
|
16349
|
+
async stagedObjectWritesByType(target) {
|
|
16350
|
+
const rows = await this.sql`
|
|
16351
|
+
SELECT object_type_id, COUNT(*) AS writes FROM (
|
|
16352
|
+
SELECT value->'row'->'ref'->>'objectTypeId' AS object_type_id
|
|
16353
|
+
FROM (${this.staged(target, "object-upsert")}) AS staged
|
|
16354
|
+
UNION ALL
|
|
16355
|
+
SELECT value->'ref'->>'objectTypeId'
|
|
16356
|
+
FROM (${this.staged(target, "object-delete")}) AS staged
|
|
16357
|
+
) AS writes
|
|
16358
|
+
GROUP BY object_type_id`;
|
|
16359
|
+
return new Map(rows.map((row) => [row.object_type_id, Number(row.writes)]));
|
|
16360
|
+
}
|
|
15827
16361
|
async stagedUniqueWork(target) {
|
|
15828
16362
|
const probe = (kind) => this.sql`
|
|
15829
16363
|
EXISTS (
|
|
@@ -16405,7 +16939,7 @@ class PgOntologyMaterializationStorage {
|
|
|
16405
16939
|
}
|
|
16406
16940
|
return session.publicSession();
|
|
16407
16941
|
} catch (error) {
|
|
16408
|
-
|
|
16942
|
+
this.sessions.discard(session);
|
|
16409
16943
|
throw error;
|
|
16410
16944
|
}
|
|
16411
16945
|
}
|
|
@@ -16459,6 +16993,17 @@ class PgOntologyMaterializationStorage {
|
|
|
16459
16993
|
}
|
|
16460
16994
|
}
|
|
16461
16995
|
}
|
|
16996
|
+
async readState(input) {
|
|
16997
|
+
const request = prepareMaterializationStateRead(input);
|
|
16998
|
+
return runPgRepeatableReadTransaction(this.sql, async (tx) => {
|
|
16999
|
+
const reader = new PgMaterializationStateReader(tx, request.projectId);
|
|
17000
|
+
return {
|
|
17001
|
+
objects: await reader.objectStates(request.objects),
|
|
17002
|
+
links: await reader.linkStates(request.links),
|
|
17003
|
+
linkScopes: await reader.linkSlotStates(request.linkScopes)
|
|
17004
|
+
};
|
|
17005
|
+
});
|
|
17006
|
+
}
|
|
16462
17007
|
async stageWork(input) {
|
|
16463
17008
|
await this.sessions.stage(input);
|
|
16464
17009
|
}
|
|
@@ -16485,7 +17030,15 @@ class PgOntologyMaterializationStorage {
|
|
|
16485
17030
|
committedAt: commit.committedAt,
|
|
16486
17031
|
...session.plan ? {} : { uniqueWork: session.uniqueWork }
|
|
16487
17032
|
});
|
|
16488
|
-
|
|
17033
|
+
if (applied.objectWrites >= STATISTICS_MINIMUM_CHANGES) {
|
|
17034
|
+
session.changedObjects = await this.writer.stagedObjectWritesByType({
|
|
17035
|
+
workTable: session.workTable,
|
|
17036
|
+
workId: session.workId,
|
|
17037
|
+
projectId: commit.projectId,
|
|
17038
|
+
commitId: commit.id,
|
|
17039
|
+
committedAt: commit.committedAt
|
|
17040
|
+
});
|
|
17041
|
+
}
|
|
16489
17042
|
session.changedLinks = applied.linkWrites;
|
|
16490
17043
|
session.changedUniqueValues = applied.uniqueValueWrites;
|
|
16491
17044
|
session.appliedEventCount = applied.eventCount;
|
|
@@ -16498,7 +17051,7 @@ class PgOntologyMaterializationStorage {
|
|
|
16498
17051
|
for (const activation of input.finalization.sourceActivations) {
|
|
16499
17052
|
await this.activateSource(session, activation);
|
|
16500
17053
|
}
|
|
16501
|
-
await
|
|
17054
|
+
await refreshObjectStatistics(this.sql, session.changedObjects);
|
|
16502
17055
|
await refreshMaterializedStatistics(this.sql, "links", session.changedLinks);
|
|
16503
17056
|
await refreshMaterializedStatistics(this.sql, "object_unique_values", session.changedUniqueValues);
|
|
16504
17057
|
const record = await this.insertCommit(session.header, input);
|
|
@@ -16828,14 +17381,36 @@ function materializationLockKeys(header) {
|
|
|
16828
17381
|
function databaseCount(value) {
|
|
16829
17382
|
return Number(value ?? 0);
|
|
16830
17383
|
}
|
|
17384
|
+
var STATISTICS_MINIMUM_CHANGES = 1000;
|
|
16831
17385
|
async function refreshMaterializedStatistics(sql, table, changed) {
|
|
16832
|
-
if (changed <
|
|
17386
|
+
if (changed < STATISTICS_MINIMUM_CHANGES)
|
|
16833
17387
|
return;
|
|
16834
17388
|
const [state] = await sql`
|
|
16835
17389
|
SELECT reltuples AS rows FROM pg_class WHERE oid=${table}::regclass`;
|
|
16836
|
-
if (!state || state.rows <= 0 || changed >= Math.max(1e4, state.rows * 0.1))
|
|
17390
|
+
if (!state || state.rows <= 0 || changed >= state.rows || changed >= Math.max(1e4, state.rows * 0.1))
|
|
16837
17391
|
await sql.unsafe(`ANALYZE ${table}`);
|
|
16838
17392
|
}
|
|
17393
|
+
async function refreshObjectStatistics(sql, changes) {
|
|
17394
|
+
const total = [...changes.values()].reduce((sum, changed) => sum + changed, 0);
|
|
17395
|
+
if (total < STATISTICS_MINIMUM_CHANGES)
|
|
17396
|
+
return;
|
|
17397
|
+
const [parent] = await sql`
|
|
17398
|
+
SELECT reltuples AS rows FROM pg_class WHERE oid = 'objects'::regclass`;
|
|
17399
|
+
if (!parent || parent.rows <= 0 || total >= Math.max(1e4, parent.rows * 0.1)) {
|
|
17400
|
+
await sql.unsafe("ANALYZE objects");
|
|
17401
|
+
return;
|
|
17402
|
+
}
|
|
17403
|
+
for (const [objectTypeId, changed] of changes) {
|
|
17404
|
+
if (changed < STATISTICS_MINIMUM_CHANGES)
|
|
17405
|
+
continue;
|
|
17406
|
+
await refreshMaterializedStatistics(sql, await objectStatisticsTable(sql, objectTypeId), changed);
|
|
17407
|
+
}
|
|
17408
|
+
}
|
|
17409
|
+
async function objectStatisticsTable(sql, objectTypeId) {
|
|
17410
|
+
const partition = objectPartitionName(objectTypeId);
|
|
17411
|
+
const [row] = await sql`SELECT to_regclass(${partition})::text AS found`;
|
|
17412
|
+
return row?.found ? partition : "objects_default";
|
|
17413
|
+
}
|
|
16839
17414
|
|
|
16840
17415
|
// src/ontology-storage/outbox.ts
|
|
16841
17416
|
import { serializeSixbFailure as serializeSixbFailure2 } from "@sixb/core/internal/errors";
|
|
@@ -18706,41 +19281,88 @@ import {
|
|
|
18706
19281
|
AiCostStorageError
|
|
18707
19282
|
} from "@sixb/core/storage";
|
|
18708
19283
|
|
|
19284
|
+
// src/ai-accounting-filters.ts
|
|
19285
|
+
function pgAccountingFilterValues(query) {
|
|
19286
|
+
return {
|
|
19287
|
+
kind: query.modelKind ?? null,
|
|
19288
|
+
source: query.source?.kind ?? null,
|
|
19289
|
+
workflow: query.source?.kind === "workflow" ? query.source.workflowId ?? null : null,
|
|
19290
|
+
mode: query.requester === undefined ? null : query.requester === "none" ? "none" : query.requester.type,
|
|
19291
|
+
id: query.requester === undefined || query.requester === "none" ? null : query.requester.id
|
|
19292
|
+
};
|
|
19293
|
+
}
|
|
19294
|
+
function pgAccountingFilterSql(start) {
|
|
19295
|
+
const [kind, source, workflow, mode, id] = [0, 1, 2, 3, 4].map((offset) => `$${start + offset}`);
|
|
19296
|
+
const sourceWorkflow = `COALESCE(workflow_node.workflow_id,
|
|
19297
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END)`;
|
|
19298
|
+
return `(${kind}::text IS NULL OR usage.model_kind = ${kind})
|
|
19299
|
+
AND (${source}::text IS NULL OR (CASE
|
|
19300
|
+
WHEN direct_agent.id IS NOT NULL THEN 'agent'
|
|
19301
|
+
WHEN ${sourceWorkflow} IS NOT NULL THEN 'workflow'
|
|
19302
|
+
WHEN execution.executor_kind = 'action' THEN 'action'
|
|
19303
|
+
ELSE 'other'
|
|
19304
|
+
END) = ${source})
|
|
19305
|
+
AND (${workflow}::text IS NULL OR ${sourceWorkflow} = ${workflow})
|
|
19306
|
+
AND (
|
|
19307
|
+
${mode}::text IS NULL
|
|
19308
|
+
OR (${mode} = 'none' AND execution.requested_by_user_id IS NULL
|
|
19309
|
+
AND execution.requested_by_service_account_id IS NULL)
|
|
19310
|
+
OR (${mode} = 'user' AND execution.requested_by_user_id = ${id})
|
|
19311
|
+
OR (${mode} = 'serviceAccount' AND execution.requested_by_service_account_id = ${id})
|
|
19312
|
+
)`;
|
|
19313
|
+
}
|
|
19314
|
+
|
|
18709
19315
|
// src/ai-cost-groups.ts
|
|
18710
19316
|
import {
|
|
18711
19317
|
aiModelCallGroupFragmentFromRow,
|
|
18712
19318
|
buildAiModelCallGroupsFromFragments,
|
|
18713
19319
|
normalizeAiModelCallAccountingQuery,
|
|
19320
|
+
normalizeAiModelCallGroupSort,
|
|
18714
19321
|
TOKENLESS_MODEL_KINDS
|
|
18715
19322
|
} from "@sixb/core/internal/ai-cost-storage-provider";
|
|
18716
19323
|
async function listAiModelCallGroups(sql, input) {
|
|
18717
19324
|
const query = normalizeAiModelCallAccountingQuery(input);
|
|
19325
|
+
const sort = normalizeAiModelCallGroupSort(input.sort);
|
|
19326
|
+
const filter = pgAccountingFilterValues(query);
|
|
18718
19327
|
const parameters = [
|
|
18719
19328
|
query.projectId,
|
|
18720
19329
|
query.from.toISOString(),
|
|
18721
19330
|
query.to.toISOString(),
|
|
18722
19331
|
query.providerId ?? null,
|
|
18723
19332
|
query.modelId ?? null,
|
|
18724
|
-
query.valuationStatus ?? null
|
|
19333
|
+
query.valuationStatus ?? null,
|
|
19334
|
+
filter.kind,
|
|
19335
|
+
filter.source,
|
|
19336
|
+
filter.workflow,
|
|
19337
|
+
filter.mode,
|
|
19338
|
+
filter.id
|
|
18725
19339
|
];
|
|
18726
19340
|
const [count] = await sql.unsafe(COUNT_SQL, parameters);
|
|
18727
19341
|
const rows = await sql.unsafe(PAGE_SQL, [
|
|
18728
19342
|
...parameters,
|
|
18729
19343
|
query.limit,
|
|
18730
|
-
query.offset
|
|
19344
|
+
query.offset,
|
|
19345
|
+
sort
|
|
18731
19346
|
]);
|
|
18732
19347
|
const total = Number(count?.total);
|
|
18733
19348
|
if (!Number.isSafeInteger(total) || total < 0)
|
|
18734
19349
|
throw new RangeError("[Sixb] Invalid model-call group count.");
|
|
18735
|
-
const items = buildAiModelCallGroupsFromFragments(rows.map(aiModelCallGroupFragmentFromRow));
|
|
19350
|
+
const items = buildAiModelCallGroupsFromFragments(rows.map(aiModelCallGroupFragmentFromRow), sort);
|
|
18736
19351
|
return { items, total, hasMore: query.offset + items.length < total };
|
|
18737
19352
|
}
|
|
18738
19353
|
var TOKENLESS_KINDS = TOKENLESS_MODEL_KINDS.map((kind) => `'${kind}'`).join(", ");
|
|
18739
19354
|
var FILTERED_SQL = `
|
|
18740
19355
|
WITH filtered AS (
|
|
18741
19356
|
SELECT COALESCE(root_agent.execution_id, usage.execution_id) AS root_execution_id,
|
|
19357
|
+
-- What the call is listed under, as aiModelCallGroupKey computes it.
|
|
19358
|
+
CASE
|
|
19359
|
+
WHEN root_agent.thread_id IS NOT NULL THEN 'thread:' || root_agent.thread_id
|
|
19360
|
+
WHEN workflow_node.workflow_run_id IS NOT NULL THEN 'workflowRun:' || workflow_node.workflow_run_id
|
|
19361
|
+
ELSE COALESCE(root_agent.execution_id, usage.execution_id)
|
|
19362
|
+
END AS group_key,
|
|
18742
19363
|
usage.execution_id, usage.provider_id, usage.requested_model_id, usage.occurred_at,
|
|
18743
|
-
usage.total_tokens, usage.
|
|
19364
|
+
usage.total_tokens, usage.input_tokens, usage.cache_read_input_tokens,
|
|
19365
|
+
usage.model_kind, usage.output_images, usage.input_characters,
|
|
18744
19366
|
usage.audio_duration_ms,
|
|
18745
19367
|
execution.executor_kind, execution.executor_id, execution.authority_primitive_id AS primitive_id,
|
|
18746
19368
|
COALESCE(cost.status, 'unvalued') AS valuation_status,
|
|
@@ -18753,7 +19375,8 @@ WITH filtered AS (
|
|
|
18753
19375
|
workflow_node.node_id AS workflow_agent_step_id,
|
|
18754
19376
|
workflow_agent.node_run_id AS workflow_node_run_id,
|
|
18755
19377
|
workflow_node.workflow_id AS workflow_id,
|
|
18756
|
-
workflow_node.workflow_run_id AS workflow_run_id
|
|
19378
|
+
workflow_node.workflow_run_id AS workflow_run_id,
|
|
19379
|
+
execution.requested_by_user_id, execution.requested_by_service_account_id
|
|
18757
19380
|
FROM ai_model_call_usage AS usage
|
|
18758
19381
|
JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
|
|
18759
19382
|
LEFT JOIN agent_runs AS direct_agent
|
|
@@ -18773,23 +19396,28 @@ WITH filtered AS (
|
|
|
18773
19396
|
AND ($4::text IS NULL OR usage.provider_id = $4)
|
|
18774
19397
|
AND ($5::text IS NULL OR usage.requested_model_id = $5)
|
|
18775
19398
|
AND ($6::text IS NULL OR COALESCE(cost.status, 'unvalued') = $6)
|
|
19399
|
+
AND ${pgAccountingFilterSql(7)}
|
|
18776
19400
|
), roots AS (
|
|
18777
|
-
SELECT
|
|
18778
|
-
FROM filtered GROUP BY
|
|
19401
|
+
SELECT group_key, MAX(occurred_at) AS last_call_at, SUM(COALESCE(amount_nanos, 0)) AS cost_nanos
|
|
19402
|
+
FROM filtered GROUP BY group_key
|
|
18779
19403
|
)
|
|
18780
19404
|
`;
|
|
18781
19405
|
var COUNT_SQL = `${FILTERED_SQL} SELECT CAST(COUNT(*) AS TEXT) AS total FROM roots`;
|
|
18782
19406
|
var PAGE_SQL = `${FILTERED_SQL}, page AS (
|
|
18783
|
-
SELECT
|
|
18784
|
-
ORDER BY
|
|
19407
|
+
SELECT group_key FROM roots
|
|
19408
|
+
ORDER BY CASE WHEN $14::text = 'cost' THEN cost_nanos END DESC NULLS LAST,
|
|
19409
|
+
last_call_at DESC, group_key COLLATE "C" ASC
|
|
19410
|
+
LIMIT $12 OFFSET $13
|
|
18785
19411
|
)
|
|
18786
|
-
SELECT filtered.root_execution_id, execution_id, provider_id, requested_model_id,
|
|
19412
|
+
SELECT filtered.group_key, root_execution_id, execution_id, provider_id, requested_model_id,
|
|
18787
19413
|
MIN(occurred_at) AS first_call_at, MAX(occurred_at) AS last_call_at,
|
|
18788
19414
|
CAST(COUNT(*) AS TEXT) AS call_count,
|
|
18789
19415
|
CAST(COUNT(total_tokens) AS TEXT) AS token_count,
|
|
18790
19416
|
CAST(COUNT(CASE WHEN total_tokens IS NULL AND model_kind IN (${TOKENLESS_KINDS}) THEN 1 END) AS TEXT)
|
|
18791
19417
|
AS tokenless_count,
|
|
18792
19418
|
CAST(SUM(total_tokens) AS TEXT) AS total_tokens,
|
|
19419
|
+
CAST(SUM(input_tokens) AS TEXT) AS input_tokens,
|
|
19420
|
+
CAST(SUM(cache_read_input_tokens) AS TEXT) AS cache_read_input_tokens,
|
|
18793
19421
|
CAST(SUM(output_images) AS TEXT) AS output_images,
|
|
18794
19422
|
CAST(SUM(input_characters) AS TEXT) AS input_characters,
|
|
18795
19423
|
CAST(SUM(audio_duration_ms) AS TEXT) AS audio_duration_ms,
|
|
@@ -18806,9 +19434,11 @@ SELECT filtered.root_execution_id, execution_id, provider_id, requested_model_id
|
|
|
18806
19434
|
MAX(workflow_agent_step_id) AS workflow_agent_step_id,
|
|
18807
19435
|
MAX(workflow_node_run_id) AS workflow_node_run_id,
|
|
18808
19436
|
MAX(workflow_id) AS workflow_id,
|
|
18809
|
-
MAX(workflow_run_id) AS workflow_run_id
|
|
18810
|
-
|
|
18811
|
-
|
|
19437
|
+
MAX(workflow_run_id) AS workflow_run_id,
|
|
19438
|
+
MAX(requested_by_user_id) AS requested_by_user_id,
|
|
19439
|
+
MAX(requested_by_service_account_id) AS requested_by_service_account_id
|
|
19440
|
+
FROM filtered JOIN page ON page.group_key = filtered.group_key
|
|
19441
|
+
GROUP BY filtered.group_key, root_execution_id, execution_id, provider_id, requested_model_id,
|
|
18812
19442
|
valuation_status, amount_currency
|
|
18813
19443
|
`;
|
|
18814
19444
|
|
|
@@ -19179,6 +19809,7 @@ class PgAiCostStorage {
|
|
|
19179
19809
|
const query = normalizeAiAccountingQuery(input);
|
|
19180
19810
|
const providerId = query.providerId ?? null;
|
|
19181
19811
|
const modelId = query.modelId ?? null;
|
|
19812
|
+
const filter = pgAccountingFilterValues(query);
|
|
19182
19813
|
const rows = await this.sql`
|
|
19183
19814
|
WITH filtered AS (
|
|
19184
19815
|
SELECT usage.provider_id,
|
|
@@ -19209,7 +19840,9 @@ class PgAiCostStorage {
|
|
|
19209
19840
|
) AS attribution_workflow_id,
|
|
19210
19841
|
cost.status AS cost_status,
|
|
19211
19842
|
cost.currency AS cost_currency,
|
|
19212
|
-
cost.amount_nanos AS cost_amount_nanos
|
|
19843
|
+
cost.amount_nanos AS cost_amount_nanos,
|
|
19844
|
+
execution.requested_by_user_id AS requester_user_id,
|
|
19845
|
+
execution.requested_by_service_account_id AS requester_service_account_id
|
|
19213
19846
|
FROM ai_model_call_usage AS usage
|
|
19214
19847
|
JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
|
|
19215
19848
|
LEFT JOIN agent_runs AS direct_agent
|
|
@@ -19227,6 +19860,26 @@ class PgAiCostStorage {
|
|
|
19227
19860
|
AND usage.occurred_at >= ${query.from} AND usage.occurred_at < ${query.to}
|
|
19228
19861
|
AND (${providerId}::text IS NULL OR usage.provider_id = ${providerId})
|
|
19229
19862
|
AND (${modelId}::text IS NULL OR usage.requested_model_id = ${modelId})
|
|
19863
|
+
AND (${filter.kind}::text IS NULL OR usage.model_kind = ${filter.kind})
|
|
19864
|
+
AND (${filter.source}::text IS NULL OR (CASE
|
|
19865
|
+
WHEN direct_agent.id IS NOT NULL THEN 'agent'
|
|
19866
|
+
WHEN COALESCE(workflow_node.workflow_id,
|
|
19867
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
|
|
19868
|
+
) IS NOT NULL THEN 'workflow'
|
|
19869
|
+
WHEN execution.executor_kind = 'action' THEN 'action'
|
|
19870
|
+
ELSE 'other'
|
|
19871
|
+
END) = ${filter.source})
|
|
19872
|
+
AND (${filter.workflow}::text IS NULL OR COALESCE(workflow_node.workflow_id,
|
|
19873
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
|
|
19874
|
+
) = ${filter.workflow})
|
|
19875
|
+
AND (
|
|
19876
|
+
${filter.mode}::text IS NULL
|
|
19877
|
+
OR (${filter.mode} = 'none' AND execution.requested_by_user_id IS NULL
|
|
19878
|
+
AND execution.requested_by_service_account_id IS NULL)
|
|
19879
|
+
OR (${filter.mode} = 'user' AND execution.requested_by_user_id = ${filter.id})
|
|
19880
|
+
OR (${filter.mode} = 'serviceAccount'
|
|
19881
|
+
AND execution.requested_by_service_account_id = ${filter.id})
|
|
19882
|
+
)
|
|
19230
19883
|
), classified AS (
|
|
19231
19884
|
SELECT filtered.*,
|
|
19232
19885
|
CASE
|
|
@@ -19257,8 +19910,27 @@ class PgAiCostStorage {
|
|
|
19257
19910
|
classified.*
|
|
19258
19911
|
FROM classified
|
|
19259
19912
|
UNION ALL
|
|
19913
|
+
SELECT 'modelSeries', provider_id, requested_model_id, NULL,
|
|
19914
|
+
CASE ${input.bucket}::text
|
|
19915
|
+
WHEN 'hour' THEN
|
|
19916
|
+
date_trunc('hour', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
|
|
19917
|
+
WHEN 'day' THEN
|
|
19918
|
+
date_trunc('day', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
|
|
19919
|
+
ELSE date_trunc('week', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
|
|
19920
|
+
END,
|
|
19921
|
+
classified.*
|
|
19922
|
+
FROM classified
|
|
19923
|
+
UNION ALL
|
|
19260
19924
|
SELECT 'model', provider_id, requested_model_id, NULL, NULL, classified.* FROM classified
|
|
19261
19925
|
UNION ALL
|
|
19926
|
+
SELECT 'person',
|
|
19927
|
+
CASE
|
|
19928
|
+
WHEN requester_user_id IS NOT NULL THEN 'user'
|
|
19929
|
+
WHEN requester_service_account_id IS NOT NULL THEN 'serviceAccount'
|
|
19930
|
+
END,
|
|
19931
|
+
COALESCE(requester_user_id, requester_service_account_id), NULL, NULL, classified.*
|
|
19932
|
+
FROM classified
|
|
19933
|
+
UNION ALL
|
|
19262
19934
|
SELECT 'kind', model_kind, NULL, NULL, NULL, classified.* FROM classified
|
|
19263
19935
|
UNION ALL
|
|
19264
19936
|
SELECT 'agent', attribution_agent_kind, attribution_workflow_id, attribution_agent_step_id, NULL, classified.*
|
|
@@ -19319,9 +19991,20 @@ class PgAiCostStorage {
|
|
|
19319
19991
|
const modelId = query.modelId ?? null;
|
|
19320
19992
|
const executionId = query.executionId ?? null;
|
|
19321
19993
|
const valuationStatus = query.valuationStatus ?? null;
|
|
19994
|
+
const filter = pgAccountingFilterValues(query);
|
|
19322
19995
|
const [count] = await this.sql`
|
|
19323
19996
|
SELECT COUNT(*)::text AS total
|
|
19324
19997
|
FROM ai_model_call_usage AS usage
|
|
19998
|
+
JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
|
|
19999
|
+
LEFT JOIN agent_runs AS direct_agent
|
|
20000
|
+
ON direct_agent.project_id = usage.project_id
|
|
20001
|
+
AND direct_agent.execution_id = usage.execution_id
|
|
20002
|
+
LEFT JOIN workflow_agent_node_runs AS workflow_agent
|
|
20003
|
+
ON workflow_agent.project_id = usage.project_id
|
|
20004
|
+
AND workflow_agent.execution_id = usage.execution_id
|
|
20005
|
+
LEFT JOIN workflow_node_runs AS workflow_node
|
|
20006
|
+
ON workflow_node.project_id = workflow_agent.project_id
|
|
20007
|
+
AND workflow_node.id = workflow_agent.node_run_id
|
|
19325
20008
|
LEFT JOIN ai_model_call_valuations AS cost
|
|
19326
20009
|
ON cost.project_id = usage.project_id AND cost.usage_record_id = usage.id
|
|
19327
20010
|
WHERE usage.project_id = ${query.projectId}
|
|
@@ -19336,6 +20019,26 @@ class PgAiCostStorage {
|
|
|
19336
20019
|
ELSE 'unvalued'
|
|
19337
20020
|
END
|
|
19338
20021
|
)
|
|
20022
|
+
AND (${filter.kind}::text IS NULL OR usage.model_kind = ${filter.kind})
|
|
20023
|
+
AND (${filter.source}::text IS NULL OR (CASE
|
|
20024
|
+
WHEN direct_agent.id IS NOT NULL THEN 'agent'
|
|
20025
|
+
WHEN COALESCE(workflow_node.workflow_id,
|
|
20026
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
|
|
20027
|
+
) IS NOT NULL THEN 'workflow'
|
|
20028
|
+
WHEN execution.executor_kind = 'action' THEN 'action'
|
|
20029
|
+
ELSE 'other'
|
|
20030
|
+
END) = ${filter.source})
|
|
20031
|
+
AND (${filter.workflow}::text IS NULL OR COALESCE(workflow_node.workflow_id,
|
|
20032
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
|
|
20033
|
+
) = ${filter.workflow})
|
|
20034
|
+
AND (
|
|
20035
|
+
${filter.mode}::text IS NULL
|
|
20036
|
+
OR (${filter.mode} = 'none' AND execution.requested_by_user_id IS NULL
|
|
20037
|
+
AND execution.requested_by_service_account_id IS NULL)
|
|
20038
|
+
OR (${filter.mode} = 'user' AND execution.requested_by_user_id = ${filter.id})
|
|
20039
|
+
OR (${filter.mode} = 'serviceAccount'
|
|
20040
|
+
AND execution.requested_by_service_account_id = ${filter.id})
|
|
20041
|
+
)
|
|
19339
20042
|
`;
|
|
19340
20043
|
const rows = await this.sql`
|
|
19341
20044
|
SELECT usage.*,
|
|
@@ -19391,6 +20094,26 @@ class PgAiCostStorage {
|
|
|
19391
20094
|
ELSE 'unvalued'
|
|
19392
20095
|
END
|
|
19393
20096
|
)
|
|
20097
|
+
AND (${filter.kind}::text IS NULL OR usage.model_kind = ${filter.kind})
|
|
20098
|
+
AND (${filter.source}::text IS NULL OR (CASE
|
|
20099
|
+
WHEN direct_agent.id IS NOT NULL THEN 'agent'
|
|
20100
|
+
WHEN COALESCE(workflow_node.workflow_id,
|
|
20101
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
|
|
20102
|
+
) IS NOT NULL THEN 'workflow'
|
|
20103
|
+
WHEN execution.executor_kind = 'action' THEN 'action'
|
|
20104
|
+
ELSE 'other'
|
|
20105
|
+
END) = ${filter.source})
|
|
20106
|
+
AND (${filter.workflow}::text IS NULL OR COALESCE(workflow_node.workflow_id,
|
|
20107
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
|
|
20108
|
+
) = ${filter.workflow})
|
|
20109
|
+
AND (
|
|
20110
|
+
${filter.mode}::text IS NULL
|
|
20111
|
+
OR (${filter.mode} = 'none' AND execution.requested_by_user_id IS NULL
|
|
20112
|
+
AND execution.requested_by_service_account_id IS NULL)
|
|
20113
|
+
OR (${filter.mode} = 'user' AND execution.requested_by_user_id = ${filter.id})
|
|
20114
|
+
OR (${filter.mode} = 'serviceAccount'
|
|
20115
|
+
AND execution.requested_by_service_account_id = ${filter.id})
|
|
20116
|
+
)
|
|
19394
20117
|
ORDER BY usage.occurred_at DESC, usage.id ASC
|
|
19395
20118
|
LIMIT ${query.limit} OFFSET ${query.offset}
|
|
19396
20119
|
`;
|
|
@@ -19419,7 +20142,11 @@ async function findUsage(sql, projectId, id) {
|
|
|
19419
20142
|
return row ? usageFromRow2(row, []) : null;
|
|
19420
20143
|
}
|
|
19421
20144
|
function aggregateFragmentFromRow(row) {
|
|
19422
|
-
const dimensionKeys = row.dimension === "model" ? { providerId: required2(row.key_1), modelId: required2(row.key_2) } : row.dimension === "
|
|
20145
|
+
const dimensionKeys = row.dimension === "model" ? { providerId: required2(row.key_1), modelId: required2(row.key_2) } : row.dimension === "modelSeries" ? {
|
|
20146
|
+
providerId: required2(row.key_1),
|
|
20147
|
+
modelId: required2(row.key_2),
|
|
20148
|
+
start: new Date(required2(row.bucket_start))
|
|
20149
|
+
} : row.dimension === "person" ? row.key_1 && row.key_2 ? { requesterType: row.key_1, requesterId: row.key_2 } : {} : row.dimension === "kind" ? withModelKind(aiAccountingModelKind(row.key_1)) : row.dimension === "agent" ? {
|
|
19423
20150
|
agentKind: required2(row.key_1),
|
|
19424
20151
|
...row.key_1 === "workflowAgent" ? { workflowId: required2(row.key_2), agentStepId: required2(row.key_3) } : {}
|
|
19425
20152
|
} : row.dimension === "workflow" ? { workflowId: required2(row.key_1) } : row.dimension === "series" ? { start: new Date(required2(row.bucket_start)) } : {};
|
|
@@ -20693,11 +21420,12 @@ async function linkThread(sql, projectId, artifactId, threadId, sharedAt) {
|
|
|
20693
21420
|
import { isIP } from "net";
|
|
20694
21421
|
import { connect as connectTls } from "tls";
|
|
20695
21422
|
import postgres from "postgres";
|
|
20696
|
-
var DEFAULT_IDLE_TIMEOUT_SECONDS =
|
|
21423
|
+
var DEFAULT_IDLE_TIMEOUT_SECONDS = 60;
|
|
20697
21424
|
var DEFAULT_CONNECT_TIMEOUT_SECONDS = 10;
|
|
20698
21425
|
function createPgClient(options) {
|
|
20699
21426
|
const connection = {
|
|
20700
|
-
search_path: options.schemaName
|
|
21427
|
+
search_path: options.schemaName,
|
|
21428
|
+
enable_incremental_sort: "off"
|
|
20701
21429
|
};
|
|
20702
21430
|
if (options.statementTimeoutMillis !== undefined) {
|
|
20703
21431
|
connection.statement_timeout = options.statementTimeoutMillis;
|
|
@@ -24186,4 +24914,4 @@ export {
|
|
|
24186
24914
|
migratePostgresStorage
|
|
24187
24915
|
};
|
|
24188
24916
|
|
|
24189
|
-
//# debugId=
|
|
24917
|
+
//# debugId=09BC0C242FA6748C64756E2164756E21
|