@sixb/pg 0.1.26 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +25 -1
  2. package/dist/agents/agent-storage.d.ts +8 -3
  3. package/dist/agents/agent-storage.d.ts.map +1 -1
  4. package/dist/agents/computers.d.ts +22 -0
  5. package/dist/agents/computers.d.ts.map +1 -0
  6. package/dist/agents/rows.d.ts +3 -0
  7. package/dist/agents/rows.d.ts.map +1 -1
  8. package/dist/agents/runs.d.ts.map +1 -1
  9. package/dist/agents/threads.d.ts +2 -1
  10. package/dist/agents/threads.d.ts.map +1 -1
  11. package/dist/agents/token-rates.d.ts +13 -0
  12. package/dist/agents/token-rates.d.ts.map +1 -0
  13. package/dist/ai-accounting-filters.d.ts +12 -0
  14. package/dist/ai-accounting-filters.d.ts.map +1 -0
  15. package/dist/ai-cost-groups.d.ts +1 -1
  16. package/dist/ai-cost-groups.d.ts.map +1 -1
  17. package/dist/index.d.ts +1 -1
  18. package/dist/index.js +886 -158
  19. package/dist/index.js.map +26 -21
  20. package/dist/migrations.d.ts.map +1 -1
  21. package/dist/objects/partitions.d.ts +10 -0
  22. package/dist/objects/partitions.d.ts.map +1 -0
  23. package/dist/objects/prepare-queries.d.ts.map +1 -1
  24. package/dist/objects/query-compiler.d.ts +5 -0
  25. package/dist/objects/query-compiler.d.ts.map +1 -1
  26. package/dist/objects/query-indexes.d.ts.map +1 -1
  27. package/dist/objects/sql-text.d.ts +3 -0
  28. package/dist/objects/sql-text.d.ts.map +1 -0
  29. package/dist/objects/text-count-index.d.ts.map +1 -1
  30. package/dist/ontology-storage/commits.d.ts.map +1 -1
  31. package/dist/ontology-storage/materialization-session.d.ts +7 -1
  32. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  33. package/dist/ontology-storage/materialization-writer.d.ts +2 -0
  34. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  35. package/dist/ontology-storage/materializations.d.ts +2 -1
  36. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  37. package/dist/pg-ai-cost-storage.d.ts.map +1 -1
  38. package/dist/pg-client.d.ts +1 -1
  39. package/dist/pg-client.d.ts.map +1 -1
  40. package/dist/storage-errors.d.ts +4 -1
  41. package/dist/storage-errors.d.ts.map +1 -1
  42. package/package.json +3 -3
  43. package/src/agents/agent-storage.ts +10 -3
  44. package/src/agents/computers.ts +95 -0
  45. package/src/agents/rows.ts +9 -1
  46. package/src/agents/runs.ts +25 -3
  47. package/src/agents/threads.ts +111 -20
  48. package/src/agents/token-rates.ts +76 -0
  49. package/src/ai-accounting-filters.ts +48 -0
  50. package/src/ai-cost-groups.ts +37 -12
  51. package/src/index.ts +1 -1
  52. package/src/migrations/084-agent-token-rates.sql +16 -0
  53. package/src/migrations/085-agent-computer-sessions.sql +27 -0
  54. package/src/migrations/086-partition-objects.sql +93 -0
  55. package/src/migrations.ts +8 -0
  56. package/src/objects/partitions.ts +163 -0
  57. package/src/objects/prepare-queries.ts +50 -2
  58. package/src/objects/query-compiler.ts +80 -66
  59. package/src/objects/query-indexes.ts +59 -46
  60. package/src/objects/sql-text.ts +8 -0
  61. package/src/objects/text-count-index.ts +17 -12
  62. package/src/ontology-storage/commits.ts +11 -2
  63. package/src/ontology-storage/materialization-session.ts +10 -1
  64. package/src/ontology-storage/materialization-writer.ts +14 -0
  65. package/src/ontology-storage/materializations.ts +81 -7
  66. package/src/pg-ai-cost-storage.ts +119 -15
  67. package/src/pg-client.ts +9 -2
  68. package/src/storage-errors.ts +28 -1
package/dist/index.js CHANGED
@@ -62,6 +62,7 @@ var LOCK_TABLES = [
62
62
  { setting: "max_locks_per_transaction", table: "lock table", note: "" }
63
63
  ];
64
64
  var OUT_OF_MEMORY = "53200";
65
+ var IN_FAILED_SQL_TRANSACTION = "25P02";
65
66
  function pgErrorCode(error) {
66
67
  if (error instanceof Error) {
67
68
  const code = error.code;
@@ -86,6 +87,10 @@ function isRetryableTransactionConflict(error) {
86
87
  const code = pgErrorCode(error);
87
88
  return code !== undefined && RETRYABLE_TRANSACTION_CONFLICT_CODES.has(code);
88
89
  }
90
+ function isLockConflict(error) {
91
+ const code = pgErrorCode(error);
92
+ return code === "55P03" || code === "40P01";
93
+ }
89
94
  function isConnectionLost(error) {
90
95
  const code = pgErrorCode(error);
91
96
  if (code !== undefined && CONNECTION_LOST_CODES.has(code)) {
@@ -98,6 +103,9 @@ function explainPgError(error) {
98
103
  if (isStorageUnavailable(error)) {
99
104
  return storageUnavailable(error);
100
105
  }
106
+ if (pgErrorCode(error) === IN_FAILED_SQL_TRANSACTION) {
107
+ return failedStatementLost(error);
108
+ }
101
109
  if (!(error instanceof Error) || pgErrorCode(error) !== OUT_OF_MEMORY) {
102
110
  return error;
103
111
  }
@@ -109,6 +117,9 @@ function explainPgError(error) {
109
117
  const { setting, table, note } = lockTable;
110
118
  return new Error(`[SixbPg] PostgreSQL aborted the transaction because its shared ${table} is full ` + `(SQLSTATE 53200 "${error.message}"). The table holds ${setting} \xD7 (max_connections + ` + `max_prepared_transactions) entries shared by all sessions.${note} Raise ${setting} ` + "(applied at server restart) or shorten the transactions that overlap.", { cause: error });
111
119
  }
120
+ function failedStatementLost(error) {
121
+ return new Error("[SixbPg] A statement was sent after an earlier statement of the same transaction failed, so " + "PostgreSQL rejected it (SQLSTATE 25P02) and the earlier error was lost. Code that handles " + "a failure inside a transaction must not send more SQL on it: let the error propagate.", { cause: error });
122
+ }
112
123
  function isStorageUnavailable(error) {
113
124
  const code = pgErrorCode(error);
114
125
  return isConnectionLost(error) || code !== undefined && CONNECT_FAILED_CODES.has(code);
@@ -248,7 +259,10 @@ function rowToThreadRecord(row) {
248
259
  sandboxParams: snapshotAgentThreadSandboxParams(typeof row.sandbox_params === "string" ? JSON.parse(row.sandbox_params) : row.sandbox_params)
249
260
  },
250
261
  ...row.runs_on == null ? {} : { runsOn: row.runs_on },
251
- ...row.computer_id == null ? {} : { computerId: row.computer_id },
262
+ ...row.computer_id == null || row.computer_name == null || row.computer_folder == null ? {} : {
263
+ computer: { id: row.computer_id, name: row.computer_name, folder: row.computer_folder }
264
+ },
265
+ ...row.moved_at == null ? {} : { movedAt: new Date(row.moved_at) },
252
266
  status: row.status,
253
267
  activeRunId: row.active_run_id,
254
268
  lastMessageAt: row.last_message_at ? new Date(row.last_message_at) : undefined,
@@ -818,6 +832,73 @@ class PgAgentContextCheckpointStore {
818
832
  }
819
833
  }
820
834
 
835
+ // src/agents/computers.ts
836
+ import { assertAgentComputerSession } from "@sixb/core/internal/agent-run-storage-provider";
837
+
838
+ class PgAgentComputerSessionStore {
839
+ sql;
840
+ constructor(sql) {
841
+ this.sql = sql;
842
+ }
843
+ async hold(input) {
844
+ assertAgentComputerSession(input, "SixbPg");
845
+ await this.sql`
846
+ INSERT INTO agent_computer_sessions (
847
+ project_id, owner_principal_type, owner_principal_id, id,
848
+ computer_id, computer_name, folder, expires_at
849
+ ) VALUES (
850
+ ${input.projectId}, ${input.ownerPrincipal.type}, ${input.ownerPrincipal.id}, ${input.id},
851
+ ${input.computer.id}, ${input.computer.name}, ${input.computer.folder}, ${input.expiresAt}
852
+ )
853
+ ON CONFLICT (project_id, owner_principal_type, owner_principal_id, id)
854
+ DO UPDATE SET
855
+ computer_id = excluded.computer_id,
856
+ computer_name = excluded.computer_name,
857
+ folder = excluded.folder,
858
+ expires_at = excluded.expires_at
859
+ `;
860
+ }
861
+ async release(input) {
862
+ const [row] = await this.sql`
863
+ DELETE FROM agent_computer_sessions
864
+ WHERE project_id = ${input.projectId}
865
+ AND owner_principal_type = ${input.ownerPrincipal.type}
866
+ AND owner_principal_id = ${input.ownerPrincipal.id}
867
+ AND id = ${input.id}
868
+ RETURNING *
869
+ `;
870
+ return row ? rowToSession(row) : null;
871
+ }
872
+ async listLive(input) {
873
+ const rows = await this.sql`
874
+ SELECT * FROM agent_computer_sessions
875
+ WHERE project_id = ${input.projectId}
876
+ AND owner_principal_type = ${input.ownerPrincipal.type}
877
+ AND owner_principal_id = ${input.ownerPrincipal.id}
878
+ AND expires_at > ${input.now}
879
+ ORDER BY id
880
+ `;
881
+ return rows.map(rowToSession);
882
+ }
883
+ async removeLapsed(input) {
884
+ const rows = await this.sql`
885
+ DELETE FROM agent_computer_sessions
886
+ WHERE project_id = ${input.projectId} AND expires_at <= ${input.now}
887
+ RETURNING *
888
+ `;
889
+ return rows.map(rowToSession);
890
+ }
891
+ }
892
+ function rowToSession(row) {
893
+ return {
894
+ projectId: row.project_id,
895
+ id: row.id,
896
+ ownerPrincipal: { type: row.owner_principal_type, id: row.owner_principal_id },
897
+ computer: { id: row.computer_id, name: row.computer_name, folder: row.folder },
898
+ expiresAt: new Date(row.expires_at)
899
+ };
900
+ }
901
+
821
902
  // src/agents/memory.ts
822
903
  import {
823
904
  agentMemoryConflicts,
@@ -1129,7 +1210,9 @@ import {
1129
1210
  assertConversationAgentRunSpec,
1130
1211
  assertConversationThreadRow,
1131
1212
  assertCreateSubagentRunInput,
1213
+ assertStartsOnComputer,
1132
1214
  assertSubagentThreadTakesRun,
1215
+ assertTurnInThreadPlace,
1133
1216
  newSubagentThreadRecord,
1134
1217
  subagentRunMatchesCreateInput
1135
1218
  } from "@sixb/core/internal/agent-run-storage-provider";
@@ -1217,7 +1300,7 @@ class PgAgentRunStore {
1217
1300
  try {
1218
1301
  return await runPgTransaction(this.sql, async (tx) => {
1219
1302
  const [thread] = await tx`
1220
- SELECT id, kind, active_run_id FROM agent_threads
1303
+ SELECT id, kind, active_run_id, runs_on FROM agent_threads
1221
1304
  WHERE project_id = ${input.projectId} AND id = ${input.threadId}
1222
1305
  FOR UPDATE
1223
1306
  `;
@@ -1228,6 +1311,7 @@ class PgAgentRunStore {
1228
1311
  if (thread.active_run_id !== null) {
1229
1312
  throw new AgentStorageError4("active_run_exists", `[SixbPg] Agent thread '${input.threadId}' already has an active run '${thread.active_run_id}'.`);
1230
1313
  }
1314
+ assertTurnInThreadPlace(thread.runs_on, input.spec, input.threadId, "SixbPg");
1231
1315
  const [row] = await tx`
1232
1316
  INSERT INTO agent_runs (
1233
1317
  project_id,
@@ -1361,7 +1445,14 @@ class PgAgentRunStore {
1361
1445
  async start(input) {
1362
1446
  const startedAt = input.startedAt ?? new Date;
1363
1447
  return runPgTransaction(this.sql, async (tx) => {
1364
- await this.lockStatus(tx, input.projectId, input.id, "queued");
1448
+ const run = await this.lockStatus(tx, input.projectId, input.id, "queued");
1449
+ if (input.onComputer) {
1450
+ const [thread] = await tx`
1451
+ SELECT computer_id AS id, computer_folder AS folder FROM agent_threads
1452
+ WHERE project_id = ${input.projectId} AND id = ${run.thread_id}
1453
+ `;
1454
+ assertStartsOnComputer(rowToRunRecord(run), thread?.id ? { id: thread.id, folder: thread.folder } : undefined, input.onComputer, "SixbPg");
1455
+ }
1365
1456
  const [row] = await tx`
1366
1457
  UPDATE agent_runs
1367
1458
  SET
@@ -1546,6 +1637,9 @@ class PgAgentRunStore {
1546
1637
  whereClauses.push(`parent_run_id = $${index++}`);
1547
1638
  params.push(input.parentRunId);
1548
1639
  }
1640
+ if (input.excludeComputerTurns) {
1641
+ whereClauses.push("spec->>'runsOn' IS DISTINCT FROM 'computer'");
1642
+ }
1549
1643
  index = appendRunListFilters(whereClauses, params, index, input, "COALESCE(started_at, created_at)");
1550
1644
  const { rows, total, hasMore } = await queryRunList({
1551
1645
  sql: this.sql,
@@ -1721,12 +1815,15 @@ function threadNotFound(input) {
1721
1815
 
1722
1816
  // src/agents/threads.ts
1723
1817
  import {
1818
+ assertAgentThreadComputer,
1724
1819
  assertAssignableComputer,
1725
- assertComputerId,
1820
+ assertComputerPlace,
1726
1821
  assertConversationThreadRow as assertConversationThreadRow2,
1727
- assertRunsOn,
1822
+ assertMovableToCloud,
1823
+ movesComputerPlace,
1728
1824
  participantsToAdd,
1729
1825
  planAgentThreadFork,
1826
+ queuedTurnInCloud,
1730
1827
  remapArtifactIds,
1731
1828
  snapshotAgentThreadSandboxParams as snapshotAgentThreadSandboxParams2,
1732
1829
  transitionAgentThreadSandbox
@@ -1756,11 +1853,14 @@ class PgAgentThreadStore {
1756
1853
  });
1757
1854
  }
1758
1855
  async create(input) {
1759
- assertRunsOn(input.runsOn, "Agent thread", "SixbPg");
1856
+ assertComputerPlace(input.runsOn, input.computer, "SixbPg");
1760
1857
  const sandboxParams = input.sandboxParams === undefined ? null : JSON.stringify(snapshotAgentThreadSandboxParams2(input.sandboxParams));
1761
1858
  const createdAt = input.createdAt ?? new Date;
1762
1859
  const updatedAt = input.updatedAt ?? createdAt;
1763
- const runsOn = input.runsOn === undefined ? { column: this.sql``, value: this.sql`` } : { column: this.sql`runs_on,`, value: this.sql`${input.runsOn},` };
1860
+ const runsOn = input.runsOn === undefined ? { column: this.sql``, value: this.sql`` } : {
1861
+ column: this.sql`runs_on, computer_id, computer_name, computer_folder,`,
1862
+ value: this.sql`${input.runsOn}, ${input.computer?.id ?? null}, ${input.computer?.name ?? null}, ${input.computer?.folder ?? null},`
1863
+ };
1764
1864
  try {
1765
1865
  const [row] = await this.sql`
1766
1866
  INSERT INTO agent_threads (
@@ -1800,7 +1900,7 @@ class PgAgentThreadStore {
1800
1900
  try {
1801
1901
  return await runPgTransaction(this.sql, async (tx) => {
1802
1902
  const [source] = await tx`
1803
- SELECT id, kind, active_run_id FROM agent_threads
1903
+ SELECT id, kind, active_run_id, runs_on, moved_at FROM agent_threads
1804
1904
  WHERE project_id = ${input.projectId} AND id = ${input.sourceThreadId}
1805
1905
  FOR SHARE
1806
1906
  `;
@@ -1830,6 +1930,7 @@ class PgAgentThreadStore {
1830
1930
  ORDER BY summarized_through_seq DESC, created_at DESC, id DESC
1831
1931
  `;
1832
1932
  const plan = planAgentThreadFork(input, {
1933
+ workspace: { runsOn: source.runs_on, movedAt: source.moved_at },
1833
1934
  ...activeRun?.trigger_message_id ? { triggerMessageId: activeRun.trigger_message_id } : {},
1834
1935
  messages: messages.map((message) => ({
1835
1936
  id: message.id,
@@ -1851,6 +1952,7 @@ class PgAgentThreadStore {
1851
1952
  status,
1852
1953
  last_message_at,
1853
1954
  message_count,
1955
+ moved_at,
1854
1956
  created_at,
1855
1957
  updated_at
1856
1958
  ) VALUES (
@@ -1864,6 +1966,7 @@ class PgAgentThreadStore {
1864
1966
  ${thread.status},
1865
1967
  ${thread.lastMessageAt ?? null},
1866
1968
  ${thread.messageCount},
1969
+ ${thread.movedAt ?? null},
1867
1970
  ${thread.createdAt},
1868
1971
  ${thread.updatedAt}
1869
1972
  )
@@ -1984,6 +2087,10 @@ class PgAgentThreadStore {
1984
2087
  )`);
1985
2088
  params.push(input.participantPrincipal.type, input.participantPrincipal.id);
1986
2089
  }
2090
+ if (input.activeOnComputer) {
2091
+ whereClauses.push(`computer_id = $${index++} AND computer_folder = $${index++} AND active_run_id IS NOT NULL`);
2092
+ params.push(input.activeOnComputer.id, input.activeOnComputer.folder);
2093
+ }
1987
2094
  const dir = input.order === "asc" ? "ASC" : "DESC";
1988
2095
  const { rows, total, hasMore } = await queryAgentList({
1989
2096
  sql: this.sql,
@@ -2047,25 +2154,77 @@ class PgAgentThreadStore {
2047
2154
  return rows.length > 0;
2048
2155
  }
2049
2156
  async assignComputer(input) {
2050
- assertComputerId(input.computerId, "SixbPg");
2051
- const [row] = await this.sql`
2052
- UPDATE agent_threads SET computer_id = ${input.computerId}
2053
- WHERE project_id = ${input.projectId}
2054
- AND id = ${input.id}
2055
- AND runs_on = 'computer'
2056
- ${input.ifUnassigned ? this.sql`AND computer_id IS NULL` : this.sql``}
2057
- RETURNING *
2058
- `;
2059
- if (row)
2060
- return rowToThreadRecord(row);
2061
- const thread = await this.getById(input);
2062
- assertAssignableComputer(thread, input.id, "SixbPg");
2063
- return thread;
2157
+ assertAgentThreadComputer(input.computer, "SixbPg");
2158
+ return runPgTransaction(this.sql, async (tx) => {
2159
+ const { thread, active } = await lockThreadAndActiveRun(tx, input);
2160
+ assertAssignableComputer(thread, active, input.computer, input.id, "SixbPg");
2161
+ const movedAt = movesComputerPlace(thread, input.computer) ? input.movedAt ?? new Date : null;
2162
+ const [assigned] = await tx`
2163
+ UPDATE agent_threads
2164
+ SET computer_id = ${input.computer.id},
2165
+ computer_name = ${input.computer.name},
2166
+ computer_folder = ${input.computer.folder},
2167
+ moved_at = COALESCE(${movedAt}::timestamptz, moved_at)
2168
+ WHERE project_id = ${input.projectId} AND id = ${input.id}
2169
+ RETURNING *
2170
+ `;
2171
+ return rowToThreadRecord(assigned);
2172
+ });
2173
+ }
2174
+ async moveToCloud(input) {
2175
+ return runPgTransaction(this.sql, async (tx) => {
2176
+ const { thread, active } = await lockThreadAndActiveRun(tx, input);
2177
+ assertMovableToCloud(thread, active, input.id, "SixbPg");
2178
+ const queued = queuedTurnInCloud(active);
2179
+ if (queued) {
2180
+ await tx`
2181
+ UPDATE agent_runs SET spec = ${JSON.stringify(queued.spec)}::text::jsonb
2182
+ WHERE project_id = ${input.projectId} AND id = ${queued.id}
2183
+ `;
2184
+ }
2185
+ const [movedThread] = await tx`
2186
+ UPDATE agent_threads
2187
+ SET runs_on = NULL, computer_id = NULL, computer_name = NULL, computer_folder = NULL,
2188
+ moved_at = ${input.movedAt ?? new Date}
2189
+ WHERE project_id = ${input.projectId} AND id = ${input.id}
2190
+ RETURNING *
2191
+ `;
2192
+ return {
2193
+ thread: rowToThreadRecord(movedThread),
2194
+ ...queued ? { queued } : {}
2195
+ };
2196
+ });
2064
2197
  }
2065
2198
  async listParticipants(input) {
2066
2199
  return selectParticipants(this.sql, input.projectId, [...new Set(input.threadIds)]);
2067
2200
  }
2068
2201
  }
2202
+ async function lockThreadAndActiveRun(tx, input) {
2203
+ for (let attempt = 0;; attempt += 1) {
2204
+ const [peek] = await tx`
2205
+ SELECT active_run_id FROM agent_threads
2206
+ WHERE project_id = ${input.projectId} AND id = ${input.id}
2207
+ `;
2208
+ const runId = peek?.active_run_id ?? null;
2209
+ const runs = runId ? await tx`
2210
+ SELECT * FROM agent_runs WHERE project_id = ${input.projectId} AND id = ${runId}
2211
+ FOR UPDATE
2212
+ ` : [];
2213
+ const [row] = await tx`
2214
+ SELECT * FROM agent_threads WHERE project_id = ${input.projectId} AND id = ${input.id}
2215
+ FOR UPDATE
2216
+ `;
2217
+ if ((row?.active_run_id ?? null) === runId) {
2218
+ return {
2219
+ thread: row ? rowToThreadRecord(row) : null,
2220
+ active: runs[0] ? rowToRunRecord(runs[0]) : null
2221
+ };
2222
+ }
2223
+ if (attempt >= 3) {
2224
+ throw new AgentStorageError6("invalid_state", `[SixbPg] Agent thread '${input.id}' kept changing turns while it moved: try again.`);
2225
+ }
2226
+ }
2227
+ }
2069
2228
  async function selectParticipants(sql, projectId, threadIds) {
2070
2229
  if (threadIds.length === 0) {
2071
2230
  return [];
@@ -2130,6 +2289,65 @@ async function remapCopiedParts(sql, projectId, messageIds, copies) {
2130
2289
  }
2131
2290
  }
2132
2291
 
2292
+ // src/agents/token-rates.ts
2293
+ import {
2294
+ AGENT_TOKEN_RATE_SAMPLES,
2295
+ assertAgentTokenRateRecord
2296
+ } from "@sixb/core/internal/agent-run-storage-provider";
2297
+ class PgAgentTokenRateStore {
2298
+ sql;
2299
+ constructor(sql) {
2300
+ this.sql = sql;
2301
+ }
2302
+ async record(input) {
2303
+ assertAgentTokenRateRecord(input, "SixbPg");
2304
+ await runPgTransaction(this.sql, async (tx) => {
2305
+ await tx`
2306
+ INSERT INTO agent_token_rates (
2307
+ project_id, run_id, provider_id, model_id, bytes, tokens, measured_at
2308
+ ) VALUES (
2309
+ ${input.projectId}, ${input.runId}, ${input.providerId}, ${input.modelId},
2310
+ ${input.bytes}, ${input.tokens}, ${input.measuredAt}
2311
+ )
2312
+ ON CONFLICT (project_id, run_id) DO UPDATE SET
2313
+ provider_id = EXCLUDED.provider_id, model_id = EXCLUDED.model_id,
2314
+ bytes = EXCLUDED.bytes, tokens = EXCLUDED.tokens, measured_at = EXCLUDED.measured_at
2315
+ `;
2316
+ await tx`
2317
+ DELETE FROM agent_token_rates
2318
+ WHERE project_id = ${input.projectId} AND provider_id = ${input.providerId}
2319
+ AND model_id = ${input.modelId}
2320
+ AND run_id NOT IN (
2321
+ SELECT run_id FROM agent_token_rates
2322
+ WHERE project_id = ${input.projectId} AND provider_id = ${input.providerId}
2323
+ AND model_id = ${input.modelId}
2324
+ ORDER BY measured_at DESC, run_id DESC
2325
+ LIMIT ${AGENT_TOKEN_RATE_SAMPLES}
2326
+ )
2327
+ `;
2328
+ });
2329
+ }
2330
+ async listLatest(input) {
2331
+ const rows = await this.sql`
2332
+ SELECT project_id, run_id, provider_id, model_id, bytes, tokens, measured_at
2333
+ FROM agent_token_rates
2334
+ WHERE project_id = ${input.projectId} AND provider_id = ${input.providerId}
2335
+ AND model_id = ${input.modelId}
2336
+ ORDER BY measured_at DESC, run_id DESC
2337
+ LIMIT ${AGENT_TOKEN_RATE_SAMPLES}
2338
+ `;
2339
+ return rows.map((row) => ({
2340
+ projectId: row.project_id,
2341
+ providerId: row.provider_id,
2342
+ modelId: row.model_id,
2343
+ runId: row.run_id,
2344
+ bytes: row.bytes,
2345
+ tokens: row.tokens,
2346
+ measuredAt: row.measured_at
2347
+ }));
2348
+ }
2349
+ }
2350
+
2133
2351
  // src/agents/agent-storage.ts
2134
2352
  class PgAgentStorage {
2135
2353
  threads;
@@ -2139,6 +2357,8 @@ class PgAgentStorage {
2139
2357
  memory;
2140
2358
  approvals;
2141
2359
  approvalModes;
2360
+ computers;
2361
+ tokenRates;
2142
2362
  constructor(options) {
2143
2363
  this.threads = new PgAgentThreadStore(options.sql);
2144
2364
  this.runs = new PgAgentRunStore(options.sql, options.executions);
@@ -2147,6 +2367,8 @@ class PgAgentStorage {
2147
2367
  this.memory = new PgAgentMemoryStore(options.sql);
2148
2368
  this.approvals = new PgAgentApprovalStore(options.sql);
2149
2369
  this.approvalModes = new PgAgentApprovalModeStore(options.sql);
2370
+ this.computers = new PgAgentComputerSessionStore(options.sql);
2371
+ this.tokenRates = new PgAgentTokenRateStore(options.sql);
2150
2372
  }
2151
2373
  }
2152
2374
  // src/auth-storage/auth-storage.ts
@@ -9801,6 +10023,151 @@ ALTER TABLE agent_threads ADD COLUMN runs_on TEXT CHECK (runs_on IS NULL OR runs
9801
10023
  ALTER TABLE agent_threads ADD COLUMN computer_id TEXT;
9802
10024
  `;
9803
10025
 
10026
+ // src/migrations/084-agent-token-rates.sql
10027
+ var _084_agent_token_rates_default = `-- What each model's tokenizer measured on a project's agent runs, one row per run: the bytes of a
10028
+ -- request's text as the Agent's context estimator weighs them, against the text tokens the
10029
+ -- provider counted. A record deletes the model's rows past its newest 20, so the table stays small.
10030
+ CREATE TABLE agent_token_rates (
10031
+ project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
10032
+ run_id TEXT NOT NULL CHECK (length(trim(run_id)) > 0),
10033
+ provider_id TEXT NOT NULL CHECK (length(trim(provider_id)) > 0),
10034
+ model_id TEXT NOT NULL CHECK (length(trim(model_id)) > 0),
10035
+ bytes INTEGER NOT NULL CHECK (bytes > 0),
10036
+ tokens INTEGER NOT NULL CHECK (tokens > 0),
10037
+ measured_at TIMESTAMPTZ NOT NULL,
10038
+ PRIMARY KEY (project_id, run_id)
10039
+ );
10040
+
10041
+ CREATE INDEX idx_agent_token_rates_model
10042
+ ON agent_token_rates (project_id, provider_id, model_id, measured_at DESC, run_id DESC);
10043
+ `;
10044
+
10045
+ // src/migrations/085-agent-computer-sessions.sql
10046
+ var _085_agent_computer_sessions_default = `-- Where a computer's thread runs: the computer's name and the folder there, beside its id. A
10047
+ -- computer kept before folders were has none until its owner moves the thread.
10048
+ ALTER TABLE agent_threads ADD COLUMN computer_name TEXT;
10049
+ ALTER TABLE agent_threads ADD COLUMN computer_folder TEXT;
10050
+ UPDATE agent_threads SET computer_id = NULL WHERE computer_id IS NOT NULL;
10051
+ -- When its turns last moved to where they run now: the files earlier turns used are not there.
10052
+ ALTER TABLE agent_threads ADD COLUMN moved_at TIMESTAMPTZ;
10053
+
10054
+ -- Each open sixb on its owner's computers, a lease it renews: while one is live, the threads that
10055
+ -- run on its computer in its folder are reachable there.
10056
+ CREATE TABLE agent_computer_sessions (
10057
+ project_id TEXT NOT NULL,
10058
+ owner_principal_type TEXT NOT NULL CHECK (owner_principal_type IN ('user', 'serviceAccount', 'system')),
10059
+ owner_principal_id TEXT NOT NULL CHECK (length(owner_principal_id) > 0),
10060
+ id TEXT NOT NULL CHECK (length(id) > 0),
10061
+ computer_id TEXT NOT NULL,
10062
+ computer_name TEXT NOT NULL,
10063
+ folder TEXT NOT NULL,
10064
+ expires_at TIMESTAMPTZ NOT NULL,
10065
+ PRIMARY KEY (project_id, owner_principal_type, owner_principal_id, id)
10066
+ );
10067
+ CREATE INDEX agent_computer_sessions_by_expiry ON agent_computer_sessions (project_id, expires_at);
10068
+
10069
+ -- Each open sixb asks every 20 seconds which of its owner's threads here have a turn active.
10070
+ CREATE INDEX agent_threads_active_on_computer
10071
+ ON agent_threads (project_id, owner_principal_type, owner_principal_id, computer_id, computer_folder)
10072
+ WHERE active_run_id IS NOT NULL;
10073
+ `;
10074
+
10075
+ // src/migrations/086-partition-objects.sql
10076
+ var _086_partition_objects_default = `-- Partition objects by object type, so a query plans against its own type's indexes alone instead
10077
+ -- of every index of every type. Query indexes and extended statistics go with the old table:
10078
+ -- preparation builds them again on the new partitions, right after this migration.
10079
+ DO $$
10080
+ DECLARE
10081
+ item record;
10082
+ columns_list text;
10083
+ restore_constraints text[] := '{}';
10084
+ restore_indexes text[] := '{}';
10085
+ statement text;
10086
+ BEGIN
10087
+ -- Copying the rows takes as long as it takes, whatever timeout this role's queries run under.
10088
+ -- Waiting for the lock does not: while this waits, every other query waits behind it, so it gives
10089
+ -- way to a long query, and the migration fails cleanly if that query outlasts every attempt.
10090
+ PERFORM set_config('statement_timeout', '0', true);
10091
+ PERFORM set_config('lock_timeout', '3s', true);
10092
+ FOR attempt IN 1..5 LOOP
10093
+ BEGIN
10094
+ LOCK TABLE objects IN ACCESS EXCLUSIVE MODE;
10095
+ EXIT;
10096
+ EXCEPTION WHEN lock_not_available THEN
10097
+ IF attempt = 5 THEN RAISE; END IF;
10098
+ PERFORM pg_sleep(attempt * 0.5);
10099
+ END;
10100
+ END LOOP;
10101
+
10102
+ -- objects is held now. Tables that reference it may have readers, so wait for those a little
10103
+ -- longer, and fail cleanly rather than hold every query behind objects for good.
10104
+ PERFORM set_config('lock_timeout', '30s', true);
10105
+
10106
+ FOR item IN
10107
+ SELECT conrelid::regclass::text AS table_name, conname, pg_get_constraintdef(oid) AS definition
10108
+ FROM pg_constraint
10109
+ WHERE confrelid = 'objects'::regclass AND contype = 'f' AND conparentid = 0
10110
+ LOOP
10111
+ EXECUTE format('ALTER TABLE %s DROP CONSTRAINT %I', item.table_name, item.conname);
10112
+ restore_constraints := restore_constraints || format(
10113
+ 'ALTER TABLE %s ADD CONSTRAINT %I %s', item.table_name, item.conname, item.definition
10114
+ );
10115
+ END LOOP;
10116
+
10117
+ SELECT string_agg(quote_ident(attname), ', ' ORDER BY attnum) INTO columns_list
10118
+ FROM pg_attribute
10119
+ WHERE attrelid = 'objects'::regclass AND attnum > 0 AND NOT attisdropped AND attgenerated = '';
10120
+
10121
+ -- The table's general indexes stay on the parent under their names; prepared ones do not.
10122
+ FOR item IN
10123
+ SELECT c.relname AS name, pg_get_indexdef(c.oid) AS definition
10124
+ FROM pg_index i
10125
+ JOIN pg_class c ON c.oid = i.indexrelid
10126
+ WHERE i.indrelid = 'objects'::regclass
10127
+ AND NOT i.indisprimary
10128
+ AND c.relname NOT LIKE 'sixb\\_query\\_%'
10129
+ AND c.relname NOT LIKE 'sixb\\_text\\_count\\_%'
10130
+ LOOP
10131
+ restore_indexes := restore_indexes || item.definition;
10132
+ EXECUTE format('DROP INDEX %I', item.name);
10133
+ END LOOP;
10134
+
10135
+ ALTER TABLE objects RENAME TO objects_unpartitioned;
10136
+ ALTER INDEX objects_pkey RENAME TO objects_unpartitioned_pkey;
10137
+
10138
+ CREATE TABLE objects (
10139
+ LIKE objects_unpartitioned
10140
+ INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE INCLUDING COMMENTS,
10141
+ PRIMARY KEY (project_id, object_type_id, primary_id)
10142
+ ) PARTITION BY LIST (object_type_id);
10143
+
10144
+ -- Holds a type until its first preparation gives it a partition.
10145
+ CREATE TABLE objects_default PARTITION OF objects DEFAULT;
10146
+
10147
+ FOR item IN SELECT DISTINCT object_type_id AS id FROM objects_unpartitioned LOOP
10148
+ EXECUTE format(
10149
+ 'CREATE TABLE %I PARTITION OF objects FOR VALUES IN (%L)',
10150
+ 'objects_' || left(md5(item.id), 24),
10151
+ item.id
10152
+ );
10153
+ END LOOP;
10154
+
10155
+ EXECUTE format(
10156
+ 'INSERT INTO objects (%s) SELECT %s FROM objects_unpartitioned', columns_list, columns_list
10157
+ );
10158
+ DROP TABLE objects_unpartitioned;
10159
+
10160
+ FOREACH statement IN ARRAY restore_indexes LOOP
10161
+ EXECUTE statement;
10162
+ END LOOP;
10163
+ FOREACH statement IN ARRAY restore_constraints LOOP
10164
+ EXECUTE statement;
10165
+ END LOOP;
10166
+ END $$;
10167
+
10168
+ ANALYZE objects;
10169
+ `;
10170
+
9804
10171
  // src/migrations.ts
9805
10172
  var POSTGRES_STORAGE_ADAPTER_ID = "SixbPostgresStorage";
9806
10173
  function pgSql(id, sqlText) {
@@ -10050,13 +10417,122 @@ var postgresStorageMigrations = defineMigrations({
10050
10417
  pgSql("080-subagent-threads", _080_subagent_threads_default),
10051
10418
  pgSql("081-subagent-questions", _081_subagent_questions_default),
10052
10419
  pgSql("082-agent-run-cancel-requests", _082_agent_run_cancel_requests_default),
10053
- pgSql("083-agent-thread-runs-on", _083_agent_thread_runs_on_default)
10420
+ pgSql("083-agent-thread-runs-on", _083_agent_thread_runs_on_default),
10421
+ pgSql("084-agent-token-rates", _084_agent_token_rates_default),
10422
+ pgSql("085-agent-computer-sessions", _085_agent_computer_sessions_default),
10423
+ pgSql("086-partition-objects", _086_partition_objects_default)
10054
10424
  ]
10055
10425
  });
10056
10426
 
10057
10427
  // src/objects/query-indexes.ts
10428
+ import { createHash as createHash4 } from "crypto";
10429
+ import { linkEndpointObjectTypeIds as linkEndpointObjectTypeIds2 } from "@sixb/core/internal/query";
10430
+
10431
+ // src/objects/partitions.ts
10058
10432
  import { createHash as createHash3 } from "crypto";
10059
10433
 
10434
+ // src/objects/sql-text.ts
10435
+ function identifier(value) {
10436
+ return `"${value.replace(/"/g, '""')}"`;
10437
+ }
10438
+ function literal(value) {
10439
+ if (value.includes("\x00"))
10440
+ throw new Error("[SixbPg] SQL literals cannot contain NUL");
10441
+ return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`;
10442
+ }
10443
+
10444
+ // src/objects/partitions.ts
10445
+ var LOCK_TIMEOUT = "1s";
10446
+ var LOCK_ATTEMPTS = 120;
10447
+ var LOCK_RETRY_MS = 1000;
10448
+ function objectPartitionName(objectTypeId) {
10449
+ return `objects_${createHash3("md5").update(objectTypeId).digest("hex").slice(0, 24)}`;
10450
+ }
10451
+ async function ensureObjectPartition(connection, schema, objectTypeId) {
10452
+ const name = objectPartitionName(objectTypeId);
10453
+ const partition = `${identifier(schema)}.${identifier(name)}`;
10454
+ const [existing] = await connection`
10455
+ SELECT to_regclass(${partition})::text AS found`;
10456
+ if (existing?.found)
10457
+ return name;
10458
+ for (let attempt = 1;; attempt++) {
10459
+ try {
10460
+ await createObjectPartition(connection, schema, partition, objectTypeId);
10461
+ return name;
10462
+ } catch (error) {
10463
+ if (!isLockConflict(error))
10464
+ throw error;
10465
+ if (attempt === LOCK_ATTEMPTS) {
10466
+ throw new Error(`[SixbPg] Could not add a partition for object type '${objectTypeId}': other queries held their locks through ${LOCK_ATTEMPTS} attempts. Retry when they finish.`, { cause: error });
10467
+ }
10468
+ }
10469
+ await Bun.sleep(LOCK_RETRY_MS);
10470
+ }
10471
+ }
10472
+ async function createObjectPartition(connection, schema, partition, objectTypeId) {
10473
+ const parent = `${identifier(schema)}.objects`;
10474
+ const fallback = `${identifier(schema)}.objects_default`;
10475
+ const type = literal(objectTypeId);
10476
+ const hasRows = async () => {
10477
+ const [row] = await connection.unsafe(`SELECT 1 FROM ${fallback} WHERE object_type_id = ${type} LIMIT 1`);
10478
+ return row !== undefined;
10479
+ };
10480
+ const guarded = (run) => runPgTransactionOn(connection, async () => {
10481
+ await connection.unsafe(`SET LOCAL lock_timeout = '${LOCK_TIMEOUT}'`);
10482
+ await connection.unsafe("SET LOCAL statement_timeout = 0");
10483
+ await run();
10484
+ });
10485
+ let moving = false;
10486
+ await guarded(async () => {
10487
+ moving = await hasRows();
10488
+ });
10489
+ await guarded(async () => {
10490
+ const references = await referencesToObjects(connection, parent);
10491
+ await connection.unsafe(`LOCK TABLE ONLY ${parent} IN ${moving ? "ACCESS EXCLUSIVE" : "SHARE UPDATE EXCLUSIVE"} MODE`);
10492
+ for (const { table_name: table } of references) {
10493
+ await connection.unsafe(`LOCK TABLE ${table} IN SHARE ROW EXCLUSIVE MODE`);
10494
+ }
10495
+ await connection.unsafe(`LOCK TABLE ${fallback} IN ACCESS EXCLUSIVE MODE`);
10496
+ if (!moving && await hasRows()) {
10497
+ throw Object.assign(new Error("[SixbPg] A row arrived while adding its partition."), {
10498
+ code: "55P03"
10499
+ });
10500
+ }
10501
+ await connection.unsafe(`CREATE TABLE ${partition} (LIKE ${parent} INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE)`);
10502
+ if (moving)
10503
+ await moveRowsFromDefault(connection, parent, fallback, partition, type, references);
10504
+ await connection.unsafe(`ALTER TABLE ${parent} ATTACH PARTITION ${partition} FOR VALUES IN (${type})`);
10505
+ if (moving) {
10506
+ for (const reference of references) {
10507
+ await connection.unsafe(`ALTER TABLE ${reference.table_name} ADD CONSTRAINT ${identifier(reference.name)} ${reference.definition}`);
10508
+ }
10509
+ }
10510
+ });
10511
+ if (moving)
10512
+ await guarded(() => connection.unsafe(`ANALYZE ${partition}`).then(() => {
10513
+ return;
10514
+ }));
10515
+ }
10516
+ function referencesToObjects(connection, parent) {
10517
+ return connection`
10518
+ SELECT conrelid::regclass::text AS table_name, conname AS name,
10519
+ pg_get_constraintdef(oid) AS definition
10520
+ FROM pg_constraint
10521
+ WHERE confrelid = ${parent}::regclass AND contype = 'f' AND conparentid = 0`;
10522
+ }
10523
+ async function moveRowsFromDefault(connection, parent, fallback, partition, type, references) {
10524
+ for (const reference of references) {
10525
+ await connection.unsafe(`ALTER TABLE ${reference.table_name} DROP CONSTRAINT ${identifier(reference.name)}`);
10526
+ }
10527
+ const columns = await connection`
10528
+ SELECT attname AS name FROM pg_attribute
10529
+ WHERE attrelid = ${parent}::regclass AND attnum > 0 AND NOT attisdropped AND attgenerated = ''
10530
+ ORDER BY attnum`;
10531
+ const list = columns.map((column) => identifier(column.name)).join(", ");
10532
+ await connection.unsafe(`INSERT INTO ${partition} (${list}) SELECT ${list} FROM ${fallback} WHERE object_type_id = ${type}`);
10533
+ await connection.unsafe(`DELETE FROM ${fallback} WHERE object_type_id = ${type}`);
10534
+ }
10535
+
10060
10536
  // src/objects/preparation-lock.ts
10061
10537
  async function lockPgPreparation(sql, key) {
10062
10538
  let delay = 25;
@@ -10077,6 +10553,7 @@ import {
10077
10553
  import {
10078
10554
  hasVectorProfile,
10079
10555
  isVectorProfileQuery,
10556
+ linkEndpointObjectTypeIds,
10080
10557
  normalizeVector,
10081
10558
  vectorSearchCandidateLimit
10082
10559
  } from "@sixb/core/internal/query";
@@ -10227,7 +10704,7 @@ function compileObjectQueryInternal(projectId, query, ctx) {
10227
10704
  case "page":
10228
10705
  return compilePage(projectId, query.input, query.pageSize, query.pageToken, ctx);
10229
10706
  case "traverse":
10230
- return compileTraversal(projectId, query.input, query.linkId, query.direction, query.sourceObjectTypeId, ctx);
10707
+ return compileTraversal(projectId, query, ctx);
10231
10708
  case "set":
10232
10709
  return compileSet(projectId, query.op, query.inputs, ctx);
10233
10710
  case "project":
@@ -10295,7 +10772,7 @@ function compileStart(projectId, query, ctx) {
10295
10772
  const order = compileOrder(identityOrderFields());
10296
10773
  const objects = pgObjectsOfType(ctx.source, query.objectTypeId);
10297
10774
  const sql = `
10298
- SELECT *, properties AS _cursor_properties
10775
+ SELECT *
10299
10776
  FROM ${objects.table}
10300
10777
  WHERE project_id = ? AND object_type_id = ? ${objects.filter}
10301
10778
  ${ctx.startPredicate ? `AND (${ctx.startPredicate})` : ""}
@@ -10333,7 +10810,7 @@ function compileRefs(projectId, query, ctx) {
10333
10810
  FROM jsonb_array_elements(?::text::jsonb) AS ref(value)
10334
10811
  `;
10335
10812
  const sql = `
10336
- SELECT selected.*, selected.properties AS _cursor_properties
10813
+ SELECT selected.*
10337
10814
  FROM (${requested}) AS requested
10338
10815
  JOIN ${ctx.source.objectsTable} AS selected
10339
10816
  ON selected.project_id = ?
@@ -10412,7 +10889,6 @@ function compileSort(projectId, inputQuery, fields, ctx) {
10412
10889
  input.object_type_id,
10413
10890
  input.primary_id,
10414
10891
  input.properties,
10415
- input.properties AS _cursor_properties,
10416
10892
  input.created_at,
10417
10893
  input.updated_at,
10418
10894
  input.version,
@@ -10506,17 +10982,17 @@ function compilePage(projectId, inputQuery, rawPageSize, pageToken, ctx) {
10506
10982
  }
10507
10983
  };
10508
10984
  }
10509
- function compileTraversal(projectId, inputQuery, linkId, direction, sourceObjectTypeId, ctx) {
10510
- const input = compileObjectQueryInternal(projectId, inputQuery, {
10985
+ function compileTraversal(projectId, traversal, ctx) {
10986
+ const input = compileObjectQueryInternal(projectId, traversal.input, {
10511
10987
  ...exactContext(ctx),
10512
10988
  unordered: true,
10513
10989
  startPredicate: undefined,
10514
10990
  isolateSortOverflow: false
10515
10991
  });
10516
- const relation = compileTraversalRelation(projectId, input, linkId, direction, sourceObjectTypeId, ctx.source, ctx.correlatedTraversal, ctx.startPredicate);
10992
+ const relation = compileTraversalRelation(projectId, input, traversal, ctx.source, ctx.correlatedTraversal, ctx.startPredicate);
10517
10993
  const order = compileOrder(identityOrderFields());
10518
10994
  return {
10519
- sql: `SELECT selected.*, selected.properties AS _cursor_properties ${relation.sql} ${ctx.unordered ? "" : `ORDER BY ${order.sql}`} ${ctx.isolateSortOverflow ? "OFFSET 0" : ""}`,
10995
+ sql: `SELECT selected.* ${relation.sql} ${ctx.unordered ? "" : `ORDER BY ${order.sql}`} ${ctx.isolateSortOverflow ? "OFFSET 0" : ""}`,
10520
10996
  args: [...relation.args, ...ctx.unordered ? [] : order.args],
10521
10997
  totalSql: `SELECT COUNT(*)::bigint AS total ${relation.sql}`,
10522
10998
  totalArgs: relation.args,
@@ -10528,15 +11004,15 @@ function compileTraversal(projectId, inputQuery, linkId, direction, sourceObject
10528
11004
  }
10529
11005
  };
10530
11006
  }
10531
- function compileTraversalRelation(projectId, input, linkId, direction, sourceObjectTypeId, source, correlated = false, endpointPredicate) {
10532
- const incoming = direction === "incoming";
11007
+ function compileTraversalRelation(projectId, input, traversal, source, correlated = false, endpointPredicate) {
11008
+ const incoming = traversal.direction === "incoming";
10533
11009
  const parent = incoming ? "target" : "source";
10534
11010
  const endpoint = incoming ? "source" : "target";
10535
- const constrainedType = incoming ? sourceObjectTypeId : undefined;
11011
+ const endpointType = compileEndpointTypes("selected", linkEndpointObjectTypeIds(traversal));
10536
11012
  return {
10537
11013
  sql: `FROM ${source.objectsTable} AS selected
10538
11014
  WHERE selected.project_id = ?
10539
- ${constrainedType === undefined ? "" : "AND selected.object_type_id = ?"}
11015
+ ${endpointType.sql}
10540
11016
  ${endpointPredicate ? `AND (${endpointPredicate})` : ""}
10541
11017
  AND EXISTS (
10542
11018
  SELECT 1 FROM (${input.sql}) AS input
@@ -10550,12 +11026,15 @@ function compileTraversalRelation(projectId, input, linkId, direction, sourceObj
10550
11026
  AND edge.link_id = ?
10551
11027
  ${correlated ? "OFFSET 0" : ""}
10552
11028
  )`,
10553
- args: [
10554
- projectId,
10555
- ...constrainedType === undefined ? [] : [constrainedType],
10556
- ...input.args,
10557
- linkId
10558
- ]
11029
+ args: [projectId, ...endpointType.args, ...input.args, traversal.linkId]
11030
+ };
11031
+ }
11032
+ function compileEndpointTypes(alias, objectTypeIds) {
11033
+ if (!objectTypeIds?.length)
11034
+ return { sql: "", args: [] };
11035
+ return objectTypeIds.length === 1 ? { sql: `AND ${alias}.object_type_id = ?`, args: [objectTypeIds[0]] } : {
11036
+ sql: `AND ${alias}.object_type_id IN (${objectTypeIds.map(() => "?").join(", ")})`,
11037
+ args: [...objectTypeIds]
10559
11038
  };
10560
11039
  }
10561
11040
  function boundedTraversal(query, bounded = false) {
@@ -10607,7 +11086,7 @@ function compileExpand(projectId, inputQuery, expansions, ctx) {
10607
11086
  }
10608
11087
  function expandRows(input, expansions, ctx) {
10609
11088
  const expand = compileExpansionsObject(expansions, { project: "input.project_id", type: "input.object_type_id", id: "input.primary_id" }, "", ctx);
10610
- const inputOrder = compileOrder(input.order.fields, "input", "input._cursor_properties");
11089
+ const inputOrder = compileOrder(input.order.fields, "input", `input.${input.order.propertyColumn}`);
10611
11090
  return {
10612
11091
  sql: `
10613
11092
  SELECT input.*, ${expand.sql} AS _expand
@@ -10645,7 +11124,8 @@ function compileExpansionValue(expansion, parent, path, ctx) {
10645
11124
  `${parentId} = ${parent.id}`,
10646
11125
  `${edge}.link_id = ?::text`
10647
11126
  ];
10648
- const whereArgs = [expansion.linkId];
11127
+ const endpointType = compileEndpointTypes(target, expansion.endpointObjectTypeIds);
11128
+ const whereArgs = [...endpointType.args, expansion.linkId];
10649
11129
  if (incoming && expansion.sourceObjectTypeId !== undefined) {
10650
11130
  whereParts.push(`${edge}.source_type_id = ?::text`);
10651
11131
  whereArgs.push(expansion.sourceObjectTypeId);
@@ -10656,6 +11136,7 @@ function compileExpansionValue(expansion, parent, path, ctx) {
10656
11136
  ON ${target}.project_id = ${edge}.project_id
10657
11137
  AND ${target}.object_type_id = ${neighborType}
10658
11138
  AND ${target}.primary_id = ${neighborId}
11139
+ ${endpointType.sql}
10659
11140
  WHERE ${whereParts.join(" AND ")}
10660
11141
  `;
10661
11142
  if (expansion.cardinality === "one") {
@@ -10718,7 +11199,7 @@ function compileSet(projectId, op, inputs, ctx) {
10718
11199
  const order = compileOrder(identityOrderFields());
10719
11200
  return {
10720
11201
  sql: `
10721
- SELECT *, properties AS _cursor_properties
11202
+ SELECT *
10722
11203
  FROM ${ctx.source.objectsTable}
10723
11204
  WHERE 1 = 0
10724
11205
  ORDER BY ${order.sql}
@@ -10739,7 +11220,7 @@ function compileSet(projectId, op, inputs, ctx) {
10739
11220
  const order = compileOrder(identityOrderFields());
10740
11221
  const selectedOrder = compileOrder(identityOrderFields(), "selected");
10741
11222
  const sql = `
10742
- SELECT selected.*, selected.properties AS _cursor_properties
11223
+ SELECT selected.*
10743
11224
  FROM (${identities.sql}) AS ids
10744
11225
  JOIN ${ctx.source.objectsTable} AS selected
10745
11226
  ON selected.project_id = ?
@@ -10765,7 +11246,8 @@ function compileProject(projectId, inputQuery, properties, ctx) {
10765
11246
  if (!properties)
10766
11247
  return input;
10767
11248
  const projection = compileProjectionExpression(properties);
10768
- const inputOrder = compileOrder(input.order.fields, "input", "input._cursor_properties");
11249
+ const original = `input.${input.order.propertyColumn}`;
11250
+ const inputOrder = compileOrder(input.order.fields, "input", original);
10769
11251
  const outputOrder = compileOrder(input.order.fields, undefined, "_cursor_properties");
10770
11252
  return {
10771
11253
  sql: `
@@ -10774,7 +11256,7 @@ function compileProject(projectId, inputQuery, properties, ctx) {
10774
11256
  input.object_type_id,
10775
11257
  input.primary_id,
10776
11258
  ${projection.sql} AS properties,
10777
- input._cursor_properties AS _cursor_properties,
11259
+ ${original} AS _cursor_properties,
10778
11260
  input.created_at,
10779
11261
  input.updated_at,
10780
11262
  input.version,
@@ -10807,7 +11289,7 @@ function compileAggregateSource(projectId, query, source) {
10807
11289
  return compileAggregateWhere(projectId, query.input, compileTextSearchPredicate(query.query, query.fields, query.fieldsByObjectType), source);
10808
11290
  }
10809
11291
  case "traverse":
10810
- return compileAggregateTraversal(projectId, query.input, query.linkId, query.direction, query.sourceObjectTypeId, source);
11292
+ return compileAggregateTraversal(projectId, query, source);
10811
11293
  case "set":
10812
11294
  return compileAggregateSet(projectId, query.op, query.inputs, source);
10813
11295
  case "sort":
@@ -10872,8 +11354,11 @@ function compileAggregateWhere(projectId, inputQuery, predicate, source) {
10872
11354
  args: [...input.args, ...predicate.args]
10873
11355
  };
10874
11356
  }
10875
- function compileAggregateTraversal(projectId, inputQuery, linkId, direction, sourceObjectTypeId, source) {
10876
- const input = compileAggregateSource(projectId, inputQuery, source);
11357
+ function compileAggregateTraversal(projectId, traversal, source) {
11358
+ const { linkId, direction } = traversal;
11359
+ const input = compileAggregateSource(projectId, traversal.input, source);
11360
+ const endpoints = linkEndpointObjectTypeIds(traversal);
11361
+ const sourceObjectTypeId = endpoints?.length === 1 ? endpoints[0] : undefined;
10877
11362
  if (direction === "incoming" && sourceObjectTypeId !== undefined) {
10878
11363
  return {
10879
11364
  sql: `SELECT selected.project_id, selected.object_type_id, selected.primary_id, selected.properties
@@ -10894,7 +11379,7 @@ function compileAggregateTraversal(projectId, inputQuery, linkId, direction, sou
10894
11379
  args: [...input.args, linkId, sourceObjectTypeId, projectId, sourceObjectTypeId]
10895
11380
  };
10896
11381
  }
10897
- const relation = compileTraversalRelation(projectId, input, linkId, direction, sourceObjectTypeId, source);
11382
+ const relation = compileTraversalRelation(projectId, input, traversal, source);
10898
11383
  return {
10899
11384
  sql: `SELECT selected.project_id, selected.object_type_id, selected.primary_id, selected.properties ${relation.sql}`,
10900
11385
  args: relation.args
@@ -10999,7 +11484,7 @@ function compilePredicate(predicate) {
10999
11484
  case "constant":
11000
11485
  return { sql: predicate.value ? "true" : "false", args: [] };
11001
11486
  case "eq":
11002
- return predicate.scalarKind === "decimal" ? compileDecimalEqualityPredicate(predicate.propertyId, "eq", predicate.value) : compileEqualityPredicate(predicate.propertyId, "eq", predicate.value);
11487
+ return predicate.scalarKind === "decimal" ? compileDecimalEqualityPredicate(predicate.propertyId, "eq", predicate.value) : compileEqualityPredicate(predicate.propertyId, "eq", predicate.value, predicate.scalarKind);
11003
11488
  case "neq":
11004
11489
  return predicate.scalarKind === "decimal" ? compileDecimalEqualityPredicate(predicate.propertyId, "neq", predicate.value) : compileEqualityPredicate(predicate.propertyId, "neq", predicate.value);
11005
11490
  case "lt":
@@ -11025,7 +11510,7 @@ function compilePredicate(predicate) {
11025
11510
  };
11026
11511
  }
11027
11512
  case "in":
11028
- return predicate.scalarKind === "decimal" ? compileDecimalInPredicate(predicate.propertyId, predicate.values) : compileInPredicate(predicate.propertyId, predicate.values);
11513
+ return predicate.scalarKind === "decimal" ? compileDecimalInPredicate(predicate.propertyId, predicate.values) : compileInPredicate(predicate.propertyId, predicate.values, predicate.scalarKind);
11029
11514
  case "exists": {
11030
11515
  const sql = `jsonb_exists(properties, ?::text)`;
11031
11516
  const item = { sql, args: [predicate.propertyId] };
@@ -11077,13 +11562,13 @@ function negatePredicate(predicate) {
11077
11562
  args: predicate.args
11078
11563
  };
11079
11564
  }
11080
- function compileEqualityPredicate(propertyId, op, value) {
11565
+ function compileEqualityPredicate(propertyId, op, value, scalarKind) {
11081
11566
  if (value === null) {
11082
11567
  const sql = `${jsonTypeExpression()} = 'null'`;
11083
11568
  const item = { sql, args: [propertyId] };
11084
11569
  return op === "eq" ? item : negatePredicate(item);
11085
11570
  }
11086
- if (typeof value === "string") {
11571
+ if (typeof value === "string" && !(op === "eq" && hasJsonValueIndex(scalarKind))) {
11087
11572
  if (op === "eq") {
11088
11573
  return {
11089
11574
  sql: `(${jsonTypeExpression()} = 'string' AND ${jsonTextExpression()} = ?::text AND md5(${jsonTextExpression()}) = md5(?::text))`,
@@ -11106,12 +11591,13 @@ function compileEqualityPredicate(propertyId, op, value) {
11106
11591
  args: [propertyId, propertyId, propertyId, jsonbValue(value)]
11107
11592
  };
11108
11593
  }
11109
- function compileInPredicate(propertyId, values) {
11594
+ function compileInPredicate(propertyId, values, scalarKind) {
11110
11595
  if (values.length === 0)
11111
11596
  return { sql: "0 = 1", args: [] };
11112
11597
  const nonNullValues = values.filter((value) => value !== null);
11113
- const stringValues = nonNullValues.filter((value) => typeof value === "string");
11114
- const jsonbValues = nonNullValues.filter((value) => typeof value !== "string");
11598
+ const comparedAsText = (value) => typeof value === "string" && !hasJsonValueIndex(scalarKind);
11599
+ const stringValues = nonNullValues.filter(comparedAsText);
11600
+ const jsonbValues = nonNullValues.filter((value) => !comparedAsText(value));
11115
11601
  const clauses = [];
11116
11602
  const args = [];
11117
11603
  if (values.length !== nonNullValues.length) {
@@ -11401,6 +11887,9 @@ function orderedValueExpression(scalarKind, propertyColumn = "properties") {
11401
11887
  return `((${jsonTextExpression(propertyColumn)}) COLLATE "C")`;
11402
11888
  return `NULLIF(${jsonValueExpression(propertyColumn)}, 'null'::jsonb)`;
11403
11889
  }
11890
+ function hasJsonValueIndex(scalarKind) {
11891
+ return scalarKind === "date" || scalarKind === "timestamp" || scalarKind === "uuid";
11892
+ }
11404
11893
  function isTextScalarKind(scalarKind) {
11405
11894
  return scalarKind === "string" || scalarKind === "uuid";
11406
11895
  }
@@ -11467,7 +11956,7 @@ function isPlainObject(value) {
11467
11956
  // src/objects/query-indexes.ts
11468
11957
  async function ensurePgObjectQueryIndexes(sql, schema, indexes) {
11469
11958
  const definitions = indexes.map((index) => ({
11470
- table: index.kind === "incomingLinks" ? "links" : "objects",
11959
+ table: index.kind === "incomingLinks" ? "links" : objectPartitionName(index.objectTypeId),
11471
11960
  expression: compileIndex(index),
11472
11961
  index
11473
11962
  }));
@@ -11486,6 +11975,9 @@ async function ensurePgObjectQueryIndexes(sql, schema, indexes) {
11486
11975
  let connected = true;
11487
11976
  try {
11488
11977
  await lockPgPreparation(connection, `${schema}.objects.query-indexes`);
11978
+ for (const objectTypeId of new Set(indexes.flatMap((index) => index.kind === "incomingLinks" ? [] : [index.objectTypeId]))) {
11979
+ await ensureObjectPartition(connection, schema, objectTypeId);
11980
+ }
11489
11981
  for (const definition of definitions) {
11490
11982
  names.push(await createQueryIndex(connection, schema, definition));
11491
11983
  }
@@ -11501,8 +11993,9 @@ async function ensurePgObjectQueryIndexes(sql, schema, indexes) {
11501
11993
  return names;
11502
11994
  }
11503
11995
  async function createQueryIndex(connection, schema, definition) {
11504
- const name = `sixb_query_${createHash3("sha256").update(definition.table === "objects" ? definition.expression : `${definition.table}:${definition.expression}`).digest("hex").slice(0, 32)}`;
11505
- await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${identifier(schema)}.${identifier(definition.table)} ${definition.expression}`);
11996
+ const { table } = definition;
11997
+ const name = `sixb_query_${createHash4("sha256").update(`${table}:${definition.expression}`).digest("hex").slice(0, 32)}`;
11998
+ await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${identifier(schema)}.${identifier(table)} ${definition.expression}`);
11506
11999
  const [index] = await connection`
11507
12000
  SELECT i.indisvalid AS valid FROM pg_index i
11508
12001
  JOIN pg_class c ON c.oid = i.indexrelid
@@ -11512,8 +12005,7 @@ async function createQueryIndex(connection, schema, definition) {
11512
12005
  throw new Error(`[SixbPg] Query index ${schema}.${name} is invalid after an interrupted build. Drop that index concurrently and retry ensureObjectQueryIndexes().`);
11513
12006
  if (definition.index.kind === "sort") {
11514
12007
  await connection.unsafe(`COMMENT ON INDEX ${identifier(schema)}.${identifier(name)} IS ${literal(JSON.stringify(definition.index))}`);
11515
- const scope = `project_id = ${literal(definition.index.projectId)} AND object_type_id = ${literal(definition.index.objectTypeId)}`;
11516
- await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(`${name}_overflow`)} ON ${identifier(schema)}.objects (primary_id) WHERE ${scope} AND NOT (${sortIndexBound(definition.index)})`);
12008
+ await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(`${name}_overflow`)} ON ${identifier(schema)}.${identifier(table)} (primary_id) WHERE project_id = ${literal(definition.index.projectId)} AND NOT (${sortIndexBound(definition.index)})`);
11517
12009
  const superseded = await connection`
11518
12010
  SELECT c.relname AS name FROM pg_index i
11519
12011
  JOIN pg_class c ON c.oid = i.indexrelid
@@ -11529,24 +12021,33 @@ async function createQueryIndex(connection, schema, definition) {
11529
12021
  return name;
11530
12022
  }
11531
12023
  async function ensureExpressionStatistics(connection, schema, indexes) {
11532
- let statisticsChanged = false;
11533
- const expressions = new Set(indexes.flatMap(statisticsExpressions));
11534
- for (const expression of expressions) {
11535
- const name = `sixb_query_stats_${createHash3("sha256").update(expression).digest("hex").slice(0, 24)}`;
11536
- const [existing] = await connection`
11537
- SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
11538
- WHERE n.nspname = ${schema} AND s.stxname = ${name}`;
11539
- if (existing)
12024
+ const expressionsByType = new Map;
12025
+ for (const index of indexes) {
12026
+ if (index.kind === "incomingLinks")
11540
12027
  continue;
11541
- await connection.unsafe(`CREATE STATISTICS ${identifier(schema)}.${identifier(name)} ON project_id, object_type_id, ${expression} FROM ${identifier(schema)}.objects`);
11542
- statisticsChanged = true;
12028
+ const expressions = expressionsByType.get(index.objectTypeId) ?? new Set;
12029
+ for (const expression of statisticsExpressions(index))
12030
+ expressions.add(expression);
12031
+ expressionsByType.set(index.objectTypeId, expressions);
12032
+ }
12033
+ for (const [objectTypeId, expressions] of expressionsByType) {
12034
+ const partition = `${identifier(schema)}.${identifier(objectPartitionName(objectTypeId))}`;
12035
+ let statisticsChanged = false;
12036
+ for (const expression of expressions) {
12037
+ const name = `sixb_query_stats_${createHash4("sha256").update(`${objectTypeId}:${expression}`).digest("hex").slice(0, 24)}`;
12038
+ const [existing] = await connection`
12039
+ SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
12040
+ WHERE n.nspname = ${schema} AND s.stxname = ${name}`;
12041
+ if (existing)
12042
+ continue;
12043
+ await connection.unsafe(`CREATE STATISTICS ${identifier(schema)}.${identifier(name)} ON project_id, ${expression} FROM ${partition}`);
12044
+ statisticsChanged = true;
12045
+ }
12046
+ if (statisticsChanged)
12047
+ await connection.unsafe(`ANALYZE ${partition}`);
11543
12048
  }
11544
- if (statisticsChanged)
11545
- await connection.unsafe(`ANALYZE ${identifier(schema)}.objects`);
11546
12049
  }
11547
12050
  function statisticsExpressions(index) {
11548
- if (index.kind === "incomingLinks")
11549
- return [];
11550
12051
  const equality = new Set(Object.keys(index.where ?? {}));
11551
12052
  if (index.kind === "filter")
11552
12053
  for (const property of index.properties)
@@ -11570,10 +12071,7 @@ function compileIndex(index) {
11570
12071
  throw new Error("[SixbPg] Incoming link indexes do not accept object property conditions");
11571
12072
  return `(target_id, link_id, source_type_id, source_id) INCLUDE (project_id, target_type_id) WHERE project_id = ${literal(index.projectId)} AND target_type_id = ${literal(index.objectTypeId)}`;
11572
12073
  }
11573
- const predicate = [
11574
- `project_id = ${literal(index.projectId)}`,
11575
- `object_type_id = ${literal(index.objectTypeId)}`
11576
- ];
12074
+ const predicate = [`project_id = ${literal(index.projectId)}`];
11577
12075
  for (const [propertyId, value] of Object.entries(index.where ?? {}).sort(([a], [b]) => a.localeCompare(b))) {
11578
12076
  const property = `properties -> ${literal(propertyId)}`;
11579
12077
  if (value === null)
@@ -11626,14 +12124,6 @@ function sortValue(field) {
11626
12124
  return `(${text} COLLATE "C")`;
11627
12125
  return `(NULLIF(properties -> ${literal(field.propertyId)}, 'null'::jsonb))`;
11628
12126
  }
11629
- function literal(value) {
11630
- if (value.includes("\x00"))
11631
- throw new Error("[SixbPg] Query index names and values cannot contain NUL");
11632
- return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`;
11633
- }
11634
- function identifier(value) {
11635
- return `"${value.replace(/"/g, '""')}"`;
11636
- }
11637
12127
  function sortIndexBound(index) {
11638
12128
  const sizes = [
11639
12129
  "octet_length(primary_id)",
@@ -11669,8 +12159,8 @@ async function readPgSortIndexes(sql) {
11669
12159
  JOIN pg_class c ON c.oid = i.indexrelid
11670
12160
  JOIN pg_description d
11671
12161
  ON d.objoid = c.oid AND d.classoid = 'pg_class'::regclass AND d.objsubid = 0
11672
- WHERE i.indrelid = 'objects'::regclass AND i.indisvalid
11673
- AND c.relname LIKE 'sixb_query_%'
12162
+ WHERE i.indrelid IN (SELECT inhrelid FROM pg_inherits WHERE inhparent = 'objects'::regclass)
12163
+ AND i.indisvalid AND c.relname LIKE 'sixb_query_%'
11674
12164
  `;
11675
12165
  const indexes = [];
11676
12166
  for (const row of rows) {
@@ -11705,9 +12195,9 @@ async function findPgSortIndexBound(sql, catalog, projectId, query) {
11705
12195
  const inspect = (predicate) => {
11706
12196
  if (predicate.op === "and")
11707
12197
  predicate.items.forEach(inspect);
11708
- else if (predicate.op === "eq" && typeof predicate.value === "string")
12198
+ else if (predicate.op === "eq" && typeof predicate.value === "string" && !hasJsonValueIndex(predicate.scalarKind))
11709
12199
  equality.add(predicate.propertyId);
11710
- else if (predicate.op === "in" && predicate.values.length === 1 && typeof predicate.values[0] === "string")
12200
+ else if (predicate.op === "in" && predicate.values.length === 1 && typeof predicate.values[0] === "string" && !hasJsonValueIndex(predicate.scalarKind))
11711
12201
  equality.add(predicate.propertyId);
11712
12202
  };
11713
12203
  inspect(input.predicate);
@@ -11717,8 +12207,9 @@ async function findPgSortIndexBound(sql, catalog, projectId, query) {
11717
12207
  let objectTypeId;
11718
12208
  if (input.kind === "start" && !input.includeSubtypes) {
11719
12209
  objectTypeId = input.objectTypeId;
11720
- } else if (input.kind === "traverse" && input.direction === "incoming") {
11721
- objectTypeId = input.sourceObjectTypeId;
12210
+ } else if (input.kind === "traverse") {
12211
+ const endpoints = linkEndpointObjectTypeIds2(input);
12212
+ objectTypeId = endpoints?.length === 1 ? endpoints[0] : undefined;
11722
12213
  }
11723
12214
  if (!objectTypeId || !fields?.length || fields.some((field) => field.kind !== "property"))
11724
12215
  return;
@@ -12221,7 +12712,7 @@ function linkIdentity(link) {
12221
12712
  }
12222
12713
 
12223
12714
  // src/objects/text-count-index.ts
12224
- import { createHash as createHash4 } from "crypto";
12715
+ import { createHash as createHash5 } from "crypto";
12225
12716
  function describe(input) {
12226
12717
  const definition = { ...input, filters: [...new Set(input.filters ?? [])].sort() };
12227
12718
  if (definition.filters.length > 4)
@@ -12232,7 +12723,7 @@ function describe(input) {
12232
12723
  input.propertyId,
12233
12724
  ...definition.filters
12234
12725
  ])
12235
- literal2(value);
12726
+ literal(value);
12236
12727
  const key = hash(JSON.stringify([
12237
12728
  definition.projectId,
12238
12729
  definition.objectTypeId,
@@ -12244,10 +12735,10 @@ function describe(input) {
12244
12735
  property,
12245
12736
  `sixb_qfilter_${hash(JSON.stringify([input.projectId, input.objectTypeId, property]))}`
12246
12737
  ]));
12247
- const scope = `project_id = ${literal2(input.projectId)} AND object_type_id = ${literal2(input.objectTypeId)}`;
12738
+ const scope = `project_id = ${literal(input.projectId)} AND object_type_id = ${literal(input.objectTypeId)}`;
12248
12739
  const compact = [
12249
- `octet_length(${identifier2(textColumn)}) <= 1024`,
12250
- ...[...filters.values()].map((column) => `(${identifier2(column)} IS NULL OR octet_length(${identifier2(column)}) <= 128)`)
12740
+ `octet_length(${identifier(textColumn)}) <= 1024`,
12741
+ ...[...filters.values()].map((column) => `(${identifier(column)} IS NULL OR octet_length(${identifier(column)}) <= 128)`)
12251
12742
  ].join(" AND ");
12252
12743
  return { definition, name: `sixb_text_count_${key}`, textColumn, filters, scope, compact };
12253
12744
  }
@@ -12263,7 +12754,7 @@ async function preparePgObjectTextCounts(sql, schema, inputs) {
12263
12754
  `;
12264
12755
  if (!extension?.ready)
12265
12756
  throw new Error("[SixbPg] Text counts require CREATE EXTENSION pg_trgm WITH SCHEMA public");
12266
- const table = `${identifier2(schema)}.objects`;
12757
+ const table = `${identifier(schema)}.objects`;
12267
12758
  const names = [];
12268
12759
  await withReservedPgConnection(sql, async (connection) => {
12269
12760
  let connected = true;
@@ -12276,29 +12767,36 @@ async function preparePgObjectTextCounts(sql, schema, inputs) {
12276
12767
  const columns = new Set(existing.map((row) => row.attname));
12277
12768
  const additions = new Map;
12278
12769
  for (const item of definitions) {
12279
- additions.set(item.textColumn, `CASE WHEN ${item.scope} THEN lower(coalesce(properties ->> ${literal2(item.definition.propertyId)}, '')) END`);
12770
+ additions.set(item.textColumn, `CASE WHEN ${item.scope} THEN lower(coalesce(properties ->> ${literal(item.definition.propertyId)}, '')) END`);
12280
12771
  for (const [property, column] of item.filters)
12281
- additions.set(column, `CASE WHEN ${item.scope} AND jsonb_typeof(properties -> ${literal2(property)}) = 'string' THEN properties ->> ${literal2(property)} END`);
12772
+ additions.set(column, `CASE WHEN ${item.scope} AND jsonb_typeof(properties -> ${literal(property)}) = 'string' THEN properties ->> ${literal(property)} END`);
12282
12773
  }
12283
12774
  const missing = [...additions].filter(([column]) => !columns.has(column));
12284
12775
  if (missing.length > 0) {
12285
- await connection.unsafe(`ALTER TABLE ${table} ${missing.map(([column, expression]) => `ADD COLUMN ${identifier2(column)} text GENERATED ALWAYS AS (${expression}) STORED`).join(", ")}`);
12776
+ await connection.unsafe(`ALTER TABLE ${table} ${missing.map(([column, expression]) => `ADD COLUMN ${identifier(column)} text GENERATED ALWAYS AS (${expression}) STORED`).join(", ")}`);
12777
+ }
12778
+ const objectTypeIds = new Set(definitions.map((item) => item.definition.objectTypeId));
12779
+ for (const objectTypeId of objectTypeIds) {
12780
+ await ensureObjectPartition(connection, schema, objectTypeId);
12286
12781
  }
12287
12782
  for (const item of definitions) {
12288
12783
  const statements = textCountIndexStatements(item);
12784
+ const partition = `${identifier(schema)}.${identifier(objectPartitionName(item.definition.objectTypeId))}`;
12289
12785
  for (const [name, expression] of statements) {
12290
- await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier2(name)} ON ${table} ${expression}`);
12786
+ await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${partition} ${expression}`);
12291
12787
  const [valid] = await connection`
12292
12788
  SELECT indisvalid FROM pg_index
12293
- WHERE indexrelid = ${`${identifier2(schema)}.${identifier2(name)}`}::regclass
12789
+ WHERE indexrelid = ${`${identifier(schema)}.${identifier(name)}`}::regclass
12294
12790
  `;
12295
12791
  if (!valid?.indisvalid)
12296
12792
  throw new Error(`[SixbPg] Text count index ${schema}.${name} is invalid. Drop it concurrently and retry preparation.`);
12297
12793
  }
12298
- await connection.unsafe(`COMMENT ON INDEX ${identifier2(schema)}.${identifier2(item.name)} IS ${literal2(JSON.stringify(item.definition))}`);
12794
+ await connection.unsafe(`COMMENT ON INDEX ${identifier(schema)}.${identifier(item.name)} IS ${literal(JSON.stringify(item.definition))}`);
12299
12795
  names.push(item.name);
12300
12796
  }
12301
- await connection.unsafe(`ANALYZE ${table}`);
12797
+ for (const objectTypeId of objectTypeIds) {
12798
+ await connection.unsafe(`ANALYZE ${identifier(schema)}.${identifier(objectPartitionName(objectTypeId))}`);
12799
+ }
12302
12800
  } catch (error) {
12303
12801
  connected = !isConnectionLost(error);
12304
12802
  throw error;
@@ -12314,16 +12812,16 @@ function textCountIndexStatements(item) {
12314
12812
  const statements = [
12315
12813
  [
12316
12814
  item.name,
12317
- `(${[...item.filters.values(), item.textColumn].map(identifier2).join(", ")}) WHERE ${predicate}`
12815
+ `(${[...item.filters.values(), item.textColumn].map(identifier).join(", ")}) WHERE ${predicate}`
12318
12816
  ],
12319
12817
  [
12320
12818
  `${item.name}_gin`,
12321
- `USING gin (${identifier2(item.textColumn)} public.gin_trgm_ops) WHERE ${predicate}`
12819
+ `USING gin (${identifier(item.textColumn)} public.gin_trgm_ops) WHERE ${predicate}`
12322
12820
  ],
12323
12821
  [`${item.name}_overflow`, `(primary_id) WHERE ${item.scope} AND NOT (${item.compact})`]
12324
12822
  ];
12325
12823
  if (item.filters.size > 0) {
12326
- const columns = [...item.filters.values()].map(identifier2);
12824
+ const columns = [...item.filters.values()].map(identifier);
12327
12825
  const compact = columns.map((column) => `(${column} IS NOT NULL AND octet_length(${column}) <= 128)`).join(" AND ");
12328
12826
  statements.push([`${item.name}_facets`, `(${columns.join(", ")}) WHERE ${item.scope} AND (${compact})`], [`${item.name}_facet_overflow`, `(primary_id) WHERE ${item.scope} AND NOT (${compact})`], [
12329
12827
  `${item.name}_text_facets_overflow`,
@@ -12345,7 +12843,7 @@ async function compilePgIndexedTextAggregate(sql, projectId, query, facets) {
12345
12843
  const rows = await sql`
12346
12844
  SELECT c.relname AS name, obj_description(c.oid,'pg_class') AS definition
12347
12845
  FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid
12348
- WHERE i.indrelid='objects'::regclass AND i.indisvalid
12846
+ WHERE i.indrelid=to_regclass(${objectPartitionName(shape.objectTypeId)}) AND i.indisvalid
12349
12847
  AND c.relname LIKE 'sixb_text_count_%' AND obj_description(c.oid,'pg_class') IS NOT NULL`;
12350
12848
  for (const row of rows) {
12351
12849
  const definition = parseDefinition(row.definition);
@@ -12390,8 +12888,8 @@ async function compilePgIndexedTextAggregate(sql, projectId, query, facets) {
12390
12888
  broadWhere.push("false");
12391
12889
  }
12392
12890
  for (const term of terms) {
12393
- where.push(`${identifier2(item.textColumn)} LIKE ${bind(`%${term.replace(/[\\%_]/g, "\\$&")}%`)}::text`);
12394
- broadWhere.push(`position(${bind(term)}::text in ${identifier2(item.textColumn)}) > 0`);
12891
+ where.push(`${identifier(item.textColumn)} LIKE ${bind(`%${term.replace(/[\\%_]/g, "\\$&")}%`)}::text`);
12892
+ broadWhere.push(`position(${bind(term)}::text in ${identifier(item.textColumn)}) > 0`);
12395
12893
  }
12396
12894
  }
12397
12895
  const input = { projectId, query, item, where, broadWhere, args };
@@ -12402,12 +12900,12 @@ async function compilePgIndexedTextAggregate(sql, projectId, query, facets) {
12402
12900
  return;
12403
12901
  }
12404
12902
  function compileIndexedFacets({ projectId, query, item, where, broadWhere, args }, facets, hasText, bind) {
12405
- const allString = [...item.filters.values()].map((column) => `${identifier2(column)} IS NOT NULL`).join(" AND ") || "true";
12406
- const filterCompact = [...item.filters.values()].map((column) => `(${identifier2(column)} IS NOT NULL AND octet_length(${identifier2(column)}) <= 128)`).join(" AND ") || "true";
12903
+ const allString = [...item.filters.values()].map((column) => `${identifier(column)} IS NOT NULL`).join(" AND ") || "true";
12904
+ const filterCompact = [...item.filters.values()].map((column) => `(${identifier(column)} IS NOT NULL AND octet_length(${identifier(column)}) <= 128)`).join(" AND ") || "true";
12407
12905
  const compact = hasText ? `(${item.compact}) AND (${allString})` : filterCompact;
12408
12906
  const nativeWhere = hasText ? [...where, allString] : [where[0], where[1], compact, ...where.slice(3)];
12409
12907
  const nativeBroad = [...broadWhere, allString];
12410
- const fields = facets.map((facet, i) => `'string'::text AS f${i}_type, ${identifier2(item.filters.get(facet.propertyId))} AS f${i}_value`).join(", ");
12908
+ const fields = facets.map((facet, i) => `'string'::text AS f${i}_type, ${identifier(item.filters.get(facet.propertyId))} AS f${i}_value`).join(", ");
12411
12909
  const overflow = compilePgObjectQuery(projectId, query, {
12412
12910
  includeTotal: false,
12413
12911
  source: {
@@ -12485,7 +12983,7 @@ function stringPredicate(predicate, columns, bind) {
12485
12983
  const values = predicate.op === "in" ? predicate.values : [predicate.value];
12486
12984
  if (!values.every((value) => typeof value === "string"))
12487
12985
  return;
12488
- return values.length ? `${identifier2(column)} IN (${values.map((value) => `${bind(value)}::text`).join(", ")})` : "false";
12986
+ return values.length ? `${identifier(column)} IN (${values.map((value) => `${bind(value)}::text`).join(", ")})` : "false";
12489
12987
  }
12490
12988
  function parseDefinition(text) {
12491
12989
  try {
@@ -12503,15 +13001,7 @@ function parseDefinition(text) {
12503
13001
  }
12504
13002
  }
12505
13003
  function hash(value) {
12506
- return createHash4("sha256").update(value).digest("hex").slice(0, 24);
12507
- }
12508
- function identifier2(value) {
12509
- return `"${value.replace(/"/g, '""')}"`;
12510
- }
12511
- function literal2(value) {
12512
- if (value.includes("\x00"))
12513
- throw new Error("[SixbPg] Text count identifiers cannot contain NUL");
12514
- return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`;
13004
+ return createHash5("sha256").update(value).digest("hex").slice(0, 24);
12515
13005
  }
12516
13006
 
12517
13007
  // src/objects/reader.ts
@@ -13326,7 +13816,7 @@ function assertReconciliationPageLimit(limit) {
13326
13816
  }
13327
13817
  }
13328
13818
  // src/objects/prepare-queries.ts
13329
- import { createHash as createHash5 } from "crypto";
13819
+ import { createHash as createHash6 } from "crypto";
13330
13820
  async function preparePgObjectQueries(sql, schema, plan) {
13331
13821
  const fingerprint = preparationFingerprint(plan);
13332
13822
  const current = await readCompletedPreparation(sql, fingerprint);
@@ -13351,17 +13841,18 @@ async function preparePgObjectQueries(sql, schema, plan) {
13351
13841
  return result;
13352
13842
  }, unlock);
13353
13843
  await unlock();
13354
- return result;
13844
+ return result.status === "current" ? result : { ...result, warnings: [...result.warnings, ...await linkWarnings(connection, plan)] };
13355
13845
  });
13356
13846
  }
13357
13847
  function preparationFingerprint(plan) {
13358
13848
  const objectTypes = plan.objectTypes.map((type) => ({
13359
13849
  objectTypeId: type.objectTypeId,
13360
13850
  properties: [...type.properties].sort((a, b) => a.propertyId.localeCompare(b.propertyId)),
13361
- indexes: type.indexes.map(normalizeIndex).sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)))
13851
+ indexes: type.indexes.map(normalizeIndex).sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
13852
+ links: [...type.links].sort((a, b) => a.linkId.localeCompare(b.linkId))
13362
13853
  })).sort((a, b) => a.objectTypeId.localeCompare(b.objectTypeId));
13363
- const definition = { version: 2, projectId: plan.projectId, objectTypes };
13364
- return createHash5("sha256").update(JSON.stringify(definition)).digest("hex");
13854
+ const definition = { version: 3, projectId: plan.projectId, objectTypes };
13855
+ return createHash6("sha256").update(JSON.stringify(definition)).digest("hex");
13365
13856
  }
13366
13857
  function normalizeIndex(index) {
13367
13858
  if (index.kind === "sort") {
@@ -13394,6 +13885,8 @@ async function prepareStructures(sql, schema, plan) {
13394
13885
  if (indexes.some((index) => index.kind === "text") || textCounts.length > 0) {
13395
13886
  await ensureTrigrams(sql);
13396
13887
  }
13888
+ for (const type of plan.objectTypes)
13889
+ await ensureObjectPartition(sql, schema, type.objectTypeId);
13397
13890
  const names = await ensurePgObjectQueryIndexes(sql, schema, indexes);
13398
13891
  const texts = await preparePgObjectTextCounts(sql, schema, textCounts);
13399
13892
  return {
@@ -13403,6 +13896,27 @@ async function prepareStructures(sql, schema, plan) {
13403
13896
  warnings
13404
13897
  };
13405
13898
  }
13899
+ async function linkWarnings(sql, plan) {
13900
+ const declared = new Map(plan.objectTypes.flatMap((type) => type.links.map((link) => [`${type.objectTypeId}\x00${link.linkId}`, link])));
13901
+ try {
13902
+ const edges = await sql`
13903
+ SELECT source_type_id, link_id, target_type_id, COUNT(*) AS edges
13904
+ FROM links WHERE project_id = ${plan.projectId}
13905
+ GROUP BY source_type_id, link_id, target_type_id`;
13906
+ return edges.flatMap((edge) => {
13907
+ const name = `${edge.source_type_id}.${edge.link_id}`;
13908
+ const link = declared.get(`${edge.source_type_id}\x00${edge.link_id}`);
13909
+ if (!link) {
13910
+ return `${name}: ${edge.edges} stored edges to ${edge.target_type_id} belong to a link the ontology no longer declares. Queries may not return them.`;
13911
+ }
13912
+ return !link.targetObjectTypeIds || link.targetObjectTypeIds.includes(edge.target_type_id) ? [] : `${name}: ${edge.edges} stored edges point to ${edge.target_type_id}, which this link no longer allows. Queries may not return them.`;
13913
+ });
13914
+ } catch (error) {
13915
+ if (isConnectionLost(error))
13916
+ throw error;
13917
+ return [`Could not check stored links against the ontology: ${String(error)}`];
13918
+ }
13919
+ }
13406
13920
  function compilePreparationPlan(plan) {
13407
13921
  const indexes = [];
13408
13922
  const textCounts = [];
@@ -13662,12 +14176,16 @@ class PgOntologyCommitStorage {
13662
14176
  assertNonnegativeInteger(offset, "Ontology commit list offset");
13663
14177
  if (limit !== undefined)
13664
14178
  assertNonnegativeInteger(limit, "Ontology commit list limit");
14179
+ if (input.committedAt !== undefined) {
14180
+ assertTimestamp(input.committedAt, "Ontology commit list committedAt");
14181
+ }
13665
14182
  const runFilter = input.run?.kind === "action" ? sql`AND origin_kind = 'action' AND origin_run_id = ${input.run.id}` : input.run?.kind === "projection" ? sql`AND origin_run_id = ${input.run.id}
13666
14183
  AND origin_kind IN ('projection', 'telemetry')` : sql``;
14184
+ const committedAtFilter = input.committedAt === undefined ? sql`` : sql`AND committed_at = ${input.committedAt}::timestamptz`;
13667
14185
  const [{ total }] = await sql`
13668
14186
  SELECT COUNT(*) AS total
13669
14187
  FROM ontology_commits
13670
- WHERE project_id = ${input.projectId} ${runFilter}
14188
+ WHERE project_id = ${input.projectId} ${runFilter} ${committedAtFilter}
13671
14189
  `;
13672
14190
  const direction = input.order === "desc" ? sql`DESC` : sql`ASC`;
13673
14191
  const order = input.run?.kind === "projection" ? sql`CASE WHEN origin_kind = 'telemetry' THEN origin_batch_ordinal ELSE -1 END
@@ -13675,7 +14193,7 @@ class PgOntologyCommitStorage {
13675
14193
  const limitFragment = limit === undefined ? sql`` : sql`LIMIT ${limit}`;
13676
14194
  const rows = await sql`
13677
14195
  SELECT * FROM ontology_commits
13678
- WHERE project_id = ${input.projectId} ${runFilter}
14196
+ WHERE project_id = ${input.projectId} ${runFilter} ${committedAtFilter}
13679
14197
  ORDER BY ${order}
13680
14198
  ${limitFragment} OFFSET ${offset}
13681
14199
  `;
@@ -13711,6 +14229,7 @@ import {
13711
14229
  beginMaterializationVectorChanges,
13712
14230
  cardinalityViolation,
13713
14231
  invalidCorrelation as invalidCorrelation2,
14232
+ prepareMaterializationStateRead,
13714
14233
  sameNonnegativeCounts as sameCounts,
13715
14234
  uniqueSorted
13716
14235
  } from "@sixb/core/internal/ontology-storage-provider";
@@ -15415,7 +15934,7 @@ async function sourceRow(sql, projectId, sourceId, materializationId, lock = fal
15415
15934
  class PgMaterializationSessionState extends ProviderMaterializationSessionState {
15416
15935
  plan;
15417
15936
  stagedWorkCount = 0;
15418
- changedObjects = 0;
15937
+ changedObjects = new Map;
15419
15938
  changedLinks = 0;
15420
15939
  changedUniqueValues = 0;
15421
15940
  uniqueWork = { releases: false, claims: false };
@@ -15463,6 +15982,9 @@ class PgMaterializationSessions {
15463
15982
  return;
15464
15983
  if (!session.plan)
15465
15984
  await this.deleteWork(session.id);
15985
+ this.discard(session);
15986
+ }
15987
+ discard(session) {
15466
15988
  session.active = false;
15467
15989
  this.live.delete(session);
15468
15990
  this.context?.materializations.complete(session.providerToken);
@@ -15824,6 +16346,18 @@ class PgMaterializationWriter {
15824
16346
  await this.writeTouches(target);
15825
16347
  return { objectWrites, linkWrites, uniqueValueWrites, eventCount };
15826
16348
  }
16349
+ async stagedObjectWritesByType(target) {
16350
+ const rows = await this.sql`
16351
+ SELECT object_type_id, COUNT(*) AS writes FROM (
16352
+ SELECT value->'row'->'ref'->>'objectTypeId' AS object_type_id
16353
+ FROM (${this.staged(target, "object-upsert")}) AS staged
16354
+ UNION ALL
16355
+ SELECT value->'ref'->>'objectTypeId'
16356
+ FROM (${this.staged(target, "object-delete")}) AS staged
16357
+ ) AS writes
16358
+ GROUP BY object_type_id`;
16359
+ return new Map(rows.map((row) => [row.object_type_id, Number(row.writes)]));
16360
+ }
15827
16361
  async stagedUniqueWork(target) {
15828
16362
  const probe = (kind) => this.sql`
15829
16363
  EXISTS (
@@ -16405,7 +16939,7 @@ class PgOntologyMaterializationStorage {
16405
16939
  }
16406
16940
  return session.publicSession();
16407
16941
  } catch (error) {
16408
- await this.sessions.release(session);
16942
+ this.sessions.discard(session);
16409
16943
  throw error;
16410
16944
  }
16411
16945
  }
@@ -16459,6 +16993,17 @@ class PgOntologyMaterializationStorage {
16459
16993
  }
16460
16994
  }
16461
16995
  }
16996
+ async readState(input) {
16997
+ const request = prepareMaterializationStateRead(input);
16998
+ return runPgRepeatableReadTransaction(this.sql, async (tx) => {
16999
+ const reader = new PgMaterializationStateReader(tx, request.projectId);
17000
+ return {
17001
+ objects: await reader.objectStates(request.objects),
17002
+ links: await reader.linkStates(request.links),
17003
+ linkScopes: await reader.linkSlotStates(request.linkScopes)
17004
+ };
17005
+ });
17006
+ }
16462
17007
  async stageWork(input) {
16463
17008
  await this.sessions.stage(input);
16464
17009
  }
@@ -16485,7 +17030,15 @@ class PgOntologyMaterializationStorage {
16485
17030
  committedAt: commit.committedAt,
16486
17031
  ...session.plan ? {} : { uniqueWork: session.uniqueWork }
16487
17032
  });
16488
- session.changedObjects = applied.objectWrites;
17033
+ if (applied.objectWrites >= STATISTICS_MINIMUM_CHANGES) {
17034
+ session.changedObjects = await this.writer.stagedObjectWritesByType({
17035
+ workTable: session.workTable,
17036
+ workId: session.workId,
17037
+ projectId: commit.projectId,
17038
+ commitId: commit.id,
17039
+ committedAt: commit.committedAt
17040
+ });
17041
+ }
16489
17042
  session.changedLinks = applied.linkWrites;
16490
17043
  session.changedUniqueValues = applied.uniqueValueWrites;
16491
17044
  session.appliedEventCount = applied.eventCount;
@@ -16498,7 +17051,7 @@ class PgOntologyMaterializationStorage {
16498
17051
  for (const activation of input.finalization.sourceActivations) {
16499
17052
  await this.activateSource(session, activation);
16500
17053
  }
16501
- await refreshMaterializedStatistics(this.sql, "objects", session.changedObjects);
17054
+ await refreshObjectStatistics(this.sql, session.changedObjects);
16502
17055
  await refreshMaterializedStatistics(this.sql, "links", session.changedLinks);
16503
17056
  await refreshMaterializedStatistics(this.sql, "object_unique_values", session.changedUniqueValues);
16504
17057
  const record = await this.insertCommit(session.header, input);
@@ -16828,14 +17381,36 @@ function materializationLockKeys(header) {
16828
17381
  function databaseCount(value) {
16829
17382
  return Number(value ?? 0);
16830
17383
  }
17384
+ var STATISTICS_MINIMUM_CHANGES = 1000;
16831
17385
  async function refreshMaterializedStatistics(sql, table, changed) {
16832
- if (changed < 1000)
17386
+ if (changed < STATISTICS_MINIMUM_CHANGES)
16833
17387
  return;
16834
17388
  const [state] = await sql`
16835
17389
  SELECT reltuples AS rows FROM pg_class WHERE oid=${table}::regclass`;
16836
- if (!state || state.rows <= 0 || changed >= Math.max(1e4, state.rows * 0.1))
17390
+ if (!state || state.rows <= 0 || changed >= state.rows || changed >= Math.max(1e4, state.rows * 0.1))
16837
17391
  await sql.unsafe(`ANALYZE ${table}`);
16838
17392
  }
17393
+ async function refreshObjectStatistics(sql, changes) {
17394
+ const total = [...changes.values()].reduce((sum, changed) => sum + changed, 0);
17395
+ if (total < STATISTICS_MINIMUM_CHANGES)
17396
+ return;
17397
+ const [parent] = await sql`
17398
+ SELECT reltuples AS rows FROM pg_class WHERE oid = 'objects'::regclass`;
17399
+ if (!parent || parent.rows <= 0 || total >= Math.max(1e4, parent.rows * 0.1)) {
17400
+ await sql.unsafe("ANALYZE objects");
17401
+ return;
17402
+ }
17403
+ for (const [objectTypeId, changed] of changes) {
17404
+ if (changed < STATISTICS_MINIMUM_CHANGES)
17405
+ continue;
17406
+ await refreshMaterializedStatistics(sql, await objectStatisticsTable(sql, objectTypeId), changed);
17407
+ }
17408
+ }
17409
+ async function objectStatisticsTable(sql, objectTypeId) {
17410
+ const partition = objectPartitionName(objectTypeId);
17411
+ const [row] = await sql`SELECT to_regclass(${partition})::text AS found`;
17412
+ return row?.found ? partition : "objects_default";
17413
+ }
16839
17414
 
16840
17415
  // src/ontology-storage/outbox.ts
16841
17416
  import { serializeSixbFailure as serializeSixbFailure2 } from "@sixb/core/internal/errors";
@@ -18706,41 +19281,88 @@ import {
18706
19281
  AiCostStorageError
18707
19282
  } from "@sixb/core/storage";
18708
19283
 
19284
+ // src/ai-accounting-filters.ts
19285
+ function pgAccountingFilterValues(query) {
19286
+ return {
19287
+ kind: query.modelKind ?? null,
19288
+ source: query.source?.kind ?? null,
19289
+ workflow: query.source?.kind === "workflow" ? query.source.workflowId ?? null : null,
19290
+ mode: query.requester === undefined ? null : query.requester === "none" ? "none" : query.requester.type,
19291
+ id: query.requester === undefined || query.requester === "none" ? null : query.requester.id
19292
+ };
19293
+ }
19294
+ function pgAccountingFilterSql(start) {
19295
+ const [kind, source, workflow, mode, id] = [0, 1, 2, 3, 4].map((offset) => `$${start + offset}`);
19296
+ const sourceWorkflow = `COALESCE(workflow_node.workflow_id,
19297
+ CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END)`;
19298
+ return `(${kind}::text IS NULL OR usage.model_kind = ${kind})
19299
+ AND (${source}::text IS NULL OR (CASE
19300
+ WHEN direct_agent.id IS NOT NULL THEN 'agent'
19301
+ WHEN ${sourceWorkflow} IS NOT NULL THEN 'workflow'
19302
+ WHEN execution.executor_kind = 'action' THEN 'action'
19303
+ ELSE 'other'
19304
+ END) = ${source})
19305
+ AND (${workflow}::text IS NULL OR ${sourceWorkflow} = ${workflow})
19306
+ AND (
19307
+ ${mode}::text IS NULL
19308
+ OR (${mode} = 'none' AND execution.requested_by_user_id IS NULL
19309
+ AND execution.requested_by_service_account_id IS NULL)
19310
+ OR (${mode} = 'user' AND execution.requested_by_user_id = ${id})
19311
+ OR (${mode} = 'serviceAccount' AND execution.requested_by_service_account_id = ${id})
19312
+ )`;
19313
+ }
19314
+
18709
19315
  // src/ai-cost-groups.ts
18710
19316
  import {
18711
19317
  aiModelCallGroupFragmentFromRow,
18712
19318
  buildAiModelCallGroupsFromFragments,
18713
19319
  normalizeAiModelCallAccountingQuery,
19320
+ normalizeAiModelCallGroupSort,
18714
19321
  TOKENLESS_MODEL_KINDS
18715
19322
  } from "@sixb/core/internal/ai-cost-storage-provider";
18716
19323
  async function listAiModelCallGroups(sql, input) {
18717
19324
  const query = normalizeAiModelCallAccountingQuery(input);
19325
+ const sort = normalizeAiModelCallGroupSort(input.sort);
19326
+ const filter = pgAccountingFilterValues(query);
18718
19327
  const parameters = [
18719
19328
  query.projectId,
18720
19329
  query.from.toISOString(),
18721
19330
  query.to.toISOString(),
18722
19331
  query.providerId ?? null,
18723
19332
  query.modelId ?? null,
18724
- query.valuationStatus ?? null
19333
+ query.valuationStatus ?? null,
19334
+ filter.kind,
19335
+ filter.source,
19336
+ filter.workflow,
19337
+ filter.mode,
19338
+ filter.id
18725
19339
  ];
18726
19340
  const [count] = await sql.unsafe(COUNT_SQL, parameters);
18727
19341
  const rows = await sql.unsafe(PAGE_SQL, [
18728
19342
  ...parameters,
18729
19343
  query.limit,
18730
- query.offset
19344
+ query.offset,
19345
+ sort
18731
19346
  ]);
18732
19347
  const total = Number(count?.total);
18733
19348
  if (!Number.isSafeInteger(total) || total < 0)
18734
19349
  throw new RangeError("[Sixb] Invalid model-call group count.");
18735
- const items = buildAiModelCallGroupsFromFragments(rows.map(aiModelCallGroupFragmentFromRow));
19350
+ const items = buildAiModelCallGroupsFromFragments(rows.map(aiModelCallGroupFragmentFromRow), sort);
18736
19351
  return { items, total, hasMore: query.offset + items.length < total };
18737
19352
  }
18738
19353
  var TOKENLESS_KINDS = TOKENLESS_MODEL_KINDS.map((kind) => `'${kind}'`).join(", ");
18739
19354
  var FILTERED_SQL = `
18740
19355
  WITH filtered AS (
18741
19356
  SELECT COALESCE(root_agent.execution_id, usage.execution_id) AS root_execution_id,
19357
+ -- What the call is listed under, as aiModelCallGroupKey computes it.
19358
+ CASE
19359
+ WHEN root_agent.thread_id IS NOT NULL THEN 'thread:' || root_agent.thread_id
19360
+ WHEN workflow_node.workflow_run_id IS NOT NULL THEN 'workflowRun:' || workflow_node.workflow_run_id
19361
+ ELSE COALESCE(root_agent.execution_id, usage.execution_id)
19362
+ END AS group_key,
18742
19363
  usage.execution_id, usage.provider_id, usage.requested_model_id, usage.occurred_at,
18743
- usage.total_tokens, usage.model_kind, usage.output_images, usage.input_characters,
19364
+ usage.total_tokens, usage.input_tokens, usage.cache_read_input_tokens,
19365
+ usage.model_kind, usage.output_images, usage.input_characters,
18744
19366
  usage.audio_duration_ms,
18745
19367
  execution.executor_kind, execution.executor_id, execution.authority_primitive_id AS primitive_id,
18746
19368
  COALESCE(cost.status, 'unvalued') AS valuation_status,
@@ -18753,7 +19375,8 @@ WITH filtered AS (
18753
19375
  workflow_node.node_id AS workflow_agent_step_id,
18754
19376
  workflow_agent.node_run_id AS workflow_node_run_id,
18755
19377
  workflow_node.workflow_id AS workflow_id,
18756
- workflow_node.workflow_run_id AS workflow_run_id
19378
+ workflow_node.workflow_run_id AS workflow_run_id,
19379
+ execution.requested_by_user_id, execution.requested_by_service_account_id
18757
19380
  FROM ai_model_call_usage AS usage
18758
19381
  JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
18759
19382
  LEFT JOIN agent_runs AS direct_agent
@@ -18773,23 +19396,28 @@ WITH filtered AS (
18773
19396
  AND ($4::text IS NULL OR usage.provider_id = $4)
18774
19397
  AND ($5::text IS NULL OR usage.requested_model_id = $5)
18775
19398
  AND ($6::text IS NULL OR COALESCE(cost.status, 'unvalued') = $6)
19399
+ AND ${pgAccountingFilterSql(7)}
18776
19400
  ), roots AS (
18777
- SELECT root_execution_id, MAX(occurred_at) AS last_call_at
18778
- FROM filtered GROUP BY root_execution_id
19401
+ SELECT group_key, MAX(occurred_at) AS last_call_at, SUM(COALESCE(amount_nanos, 0)) AS cost_nanos
19402
+ FROM filtered GROUP BY group_key
18779
19403
  )
18780
19404
  `;
18781
19405
  var COUNT_SQL = `${FILTERED_SQL} SELECT CAST(COUNT(*) AS TEXT) AS total FROM roots`;
18782
19406
  var PAGE_SQL = `${FILTERED_SQL}, page AS (
18783
- SELECT root_execution_id FROM roots
18784
- ORDER BY last_call_at DESC, root_execution_id COLLATE "C" ASC LIMIT $7 OFFSET $8
19407
+ SELECT group_key FROM roots
19408
+ ORDER BY CASE WHEN $14::text = 'cost' THEN cost_nanos END DESC NULLS LAST,
19409
+ last_call_at DESC, group_key COLLATE "C" ASC
19410
+ LIMIT $12 OFFSET $13
18785
19411
  )
18786
- SELECT filtered.root_execution_id, execution_id, provider_id, requested_model_id,
19412
+ SELECT filtered.group_key, root_execution_id, execution_id, provider_id, requested_model_id,
18787
19413
  MIN(occurred_at) AS first_call_at, MAX(occurred_at) AS last_call_at,
18788
19414
  CAST(COUNT(*) AS TEXT) AS call_count,
18789
19415
  CAST(COUNT(total_tokens) AS TEXT) AS token_count,
18790
19416
  CAST(COUNT(CASE WHEN total_tokens IS NULL AND model_kind IN (${TOKENLESS_KINDS}) THEN 1 END) AS TEXT)
18791
19417
  AS tokenless_count,
18792
19418
  CAST(SUM(total_tokens) AS TEXT) AS total_tokens,
19419
+ CAST(SUM(input_tokens) AS TEXT) AS input_tokens,
19420
+ CAST(SUM(cache_read_input_tokens) AS TEXT) AS cache_read_input_tokens,
18793
19421
  CAST(SUM(output_images) AS TEXT) AS output_images,
18794
19422
  CAST(SUM(input_characters) AS TEXT) AS input_characters,
18795
19423
  CAST(SUM(audio_duration_ms) AS TEXT) AS audio_duration_ms,
@@ -18806,9 +19434,11 @@ SELECT filtered.root_execution_id, execution_id, provider_id, requested_model_id
18806
19434
  MAX(workflow_agent_step_id) AS workflow_agent_step_id,
18807
19435
  MAX(workflow_node_run_id) AS workflow_node_run_id,
18808
19436
  MAX(workflow_id) AS workflow_id,
18809
- MAX(workflow_run_id) AS workflow_run_id
18810
- FROM filtered JOIN page ON page.root_execution_id = filtered.root_execution_id
18811
- GROUP BY filtered.root_execution_id, execution_id, provider_id, requested_model_id,
19437
+ MAX(workflow_run_id) AS workflow_run_id,
19438
+ MAX(requested_by_user_id) AS requested_by_user_id,
19439
+ MAX(requested_by_service_account_id) AS requested_by_service_account_id
19440
+ FROM filtered JOIN page ON page.group_key = filtered.group_key
19441
+ GROUP BY filtered.group_key, root_execution_id, execution_id, provider_id, requested_model_id,
18812
19442
  valuation_status, amount_currency
18813
19443
  `;
18814
19444
 
@@ -19179,6 +19809,7 @@ class PgAiCostStorage {
19179
19809
  const query = normalizeAiAccountingQuery(input);
19180
19810
  const providerId = query.providerId ?? null;
19181
19811
  const modelId = query.modelId ?? null;
19812
+ const filter = pgAccountingFilterValues(query);
19182
19813
  const rows = await this.sql`
19183
19814
  WITH filtered AS (
19184
19815
  SELECT usage.provider_id,
@@ -19209,7 +19840,9 @@ class PgAiCostStorage {
19209
19840
  ) AS attribution_workflow_id,
19210
19841
  cost.status AS cost_status,
19211
19842
  cost.currency AS cost_currency,
19212
- cost.amount_nanos AS cost_amount_nanos
19843
+ cost.amount_nanos AS cost_amount_nanos,
19844
+ execution.requested_by_user_id AS requester_user_id,
19845
+ execution.requested_by_service_account_id AS requester_service_account_id
19213
19846
  FROM ai_model_call_usage AS usage
19214
19847
  JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
19215
19848
  LEFT JOIN agent_runs AS direct_agent
@@ -19227,6 +19860,26 @@ class PgAiCostStorage {
19227
19860
  AND usage.occurred_at >= ${query.from} AND usage.occurred_at < ${query.to}
19228
19861
  AND (${providerId}::text IS NULL OR usage.provider_id = ${providerId})
19229
19862
  AND (${modelId}::text IS NULL OR usage.requested_model_id = ${modelId})
19863
+ AND (${filter.kind}::text IS NULL OR usage.model_kind = ${filter.kind})
19864
+ AND (${filter.source}::text IS NULL OR (CASE
19865
+ WHEN direct_agent.id IS NOT NULL THEN 'agent'
19866
+ WHEN COALESCE(workflow_node.workflow_id,
19867
+ CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
19868
+ ) IS NOT NULL THEN 'workflow'
19869
+ WHEN execution.executor_kind = 'action' THEN 'action'
19870
+ ELSE 'other'
19871
+ END) = ${filter.source})
19872
+ AND (${filter.workflow}::text IS NULL OR COALESCE(workflow_node.workflow_id,
19873
+ CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
19874
+ ) = ${filter.workflow})
19875
+ AND (
19876
+ ${filter.mode}::text IS NULL
19877
+ OR (${filter.mode} = 'none' AND execution.requested_by_user_id IS NULL
19878
+ AND execution.requested_by_service_account_id IS NULL)
19879
+ OR (${filter.mode} = 'user' AND execution.requested_by_user_id = ${filter.id})
19880
+ OR (${filter.mode} = 'serviceAccount'
19881
+ AND execution.requested_by_service_account_id = ${filter.id})
19882
+ )
19230
19883
  ), classified AS (
19231
19884
  SELECT filtered.*,
19232
19885
  CASE
@@ -19257,8 +19910,27 @@ class PgAiCostStorage {
19257
19910
  classified.*
19258
19911
  FROM classified
19259
19912
  UNION ALL
19913
+ SELECT 'modelSeries', provider_id, requested_model_id, NULL,
19914
+ CASE ${input.bucket}::text
19915
+ WHEN 'hour' THEN
19916
+ date_trunc('hour', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
19917
+ WHEN 'day' THEN
19918
+ date_trunc('day', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
19919
+ ELSE date_trunc('week', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
19920
+ END,
19921
+ classified.*
19922
+ FROM classified
19923
+ UNION ALL
19260
19924
  SELECT 'model', provider_id, requested_model_id, NULL, NULL, classified.* FROM classified
19261
19925
  UNION ALL
19926
+ SELECT 'person',
19927
+ CASE
19928
+ WHEN requester_user_id IS NOT NULL THEN 'user'
19929
+ WHEN requester_service_account_id IS NOT NULL THEN 'serviceAccount'
19930
+ END,
19931
+ COALESCE(requester_user_id, requester_service_account_id), NULL, NULL, classified.*
19932
+ FROM classified
19933
+ UNION ALL
19262
19934
  SELECT 'kind', model_kind, NULL, NULL, NULL, classified.* FROM classified
19263
19935
  UNION ALL
19264
19936
  SELECT 'agent', attribution_agent_kind, attribution_workflow_id, attribution_agent_step_id, NULL, classified.*
@@ -19319,9 +19991,20 @@ class PgAiCostStorage {
19319
19991
  const modelId = query.modelId ?? null;
19320
19992
  const executionId = query.executionId ?? null;
19321
19993
  const valuationStatus = query.valuationStatus ?? null;
19994
+ const filter = pgAccountingFilterValues(query);
19322
19995
  const [count] = await this.sql`
19323
19996
  SELECT COUNT(*)::text AS total
19324
19997
  FROM ai_model_call_usage AS usage
19998
+ JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
19999
+ LEFT JOIN agent_runs AS direct_agent
20000
+ ON direct_agent.project_id = usage.project_id
20001
+ AND direct_agent.execution_id = usage.execution_id
20002
+ LEFT JOIN workflow_agent_node_runs AS workflow_agent
20003
+ ON workflow_agent.project_id = usage.project_id
20004
+ AND workflow_agent.execution_id = usage.execution_id
20005
+ LEFT JOIN workflow_node_runs AS workflow_node
20006
+ ON workflow_node.project_id = workflow_agent.project_id
20007
+ AND workflow_node.id = workflow_agent.node_run_id
19325
20008
  LEFT JOIN ai_model_call_valuations AS cost
19326
20009
  ON cost.project_id = usage.project_id AND cost.usage_record_id = usage.id
19327
20010
  WHERE usage.project_id = ${query.projectId}
@@ -19336,6 +20019,26 @@ class PgAiCostStorage {
19336
20019
  ELSE 'unvalued'
19337
20020
  END
19338
20021
  )
20022
+ AND (${filter.kind}::text IS NULL OR usage.model_kind = ${filter.kind})
20023
+ AND (${filter.source}::text IS NULL OR (CASE
20024
+ WHEN direct_agent.id IS NOT NULL THEN 'agent'
20025
+ WHEN COALESCE(workflow_node.workflow_id,
20026
+ CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
20027
+ ) IS NOT NULL THEN 'workflow'
20028
+ WHEN execution.executor_kind = 'action' THEN 'action'
20029
+ ELSE 'other'
20030
+ END) = ${filter.source})
20031
+ AND (${filter.workflow}::text IS NULL OR COALESCE(workflow_node.workflow_id,
20032
+ CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
20033
+ ) = ${filter.workflow})
20034
+ AND (
20035
+ ${filter.mode}::text IS NULL
20036
+ OR (${filter.mode} = 'none' AND execution.requested_by_user_id IS NULL
20037
+ AND execution.requested_by_service_account_id IS NULL)
20038
+ OR (${filter.mode} = 'user' AND execution.requested_by_user_id = ${filter.id})
20039
+ OR (${filter.mode} = 'serviceAccount'
20040
+ AND execution.requested_by_service_account_id = ${filter.id})
20041
+ )
19339
20042
  `;
19340
20043
  const rows = await this.sql`
19341
20044
  SELECT usage.*,
@@ -19391,6 +20094,26 @@ class PgAiCostStorage {
19391
20094
  ELSE 'unvalued'
19392
20095
  END
19393
20096
  )
20097
+ AND (${filter.kind}::text IS NULL OR usage.model_kind = ${filter.kind})
20098
+ AND (${filter.source}::text IS NULL OR (CASE
20099
+ WHEN direct_agent.id IS NOT NULL THEN 'agent'
20100
+ WHEN COALESCE(workflow_node.workflow_id,
20101
+ CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
20102
+ ) IS NOT NULL THEN 'workflow'
20103
+ WHEN execution.executor_kind = 'action' THEN 'action'
20104
+ ELSE 'other'
20105
+ END) = ${filter.source})
20106
+ AND (${filter.workflow}::text IS NULL OR COALESCE(workflow_node.workflow_id,
20107
+ CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
20108
+ ) = ${filter.workflow})
20109
+ AND (
20110
+ ${filter.mode}::text IS NULL
20111
+ OR (${filter.mode} = 'none' AND execution.requested_by_user_id IS NULL
20112
+ AND execution.requested_by_service_account_id IS NULL)
20113
+ OR (${filter.mode} = 'user' AND execution.requested_by_user_id = ${filter.id})
20114
+ OR (${filter.mode} = 'serviceAccount'
20115
+ AND execution.requested_by_service_account_id = ${filter.id})
20116
+ )
19394
20117
  ORDER BY usage.occurred_at DESC, usage.id ASC
19395
20118
  LIMIT ${query.limit} OFFSET ${query.offset}
19396
20119
  `;
@@ -19419,7 +20142,11 @@ async function findUsage(sql, projectId, id) {
19419
20142
  return row ? usageFromRow2(row, []) : null;
19420
20143
  }
19421
20144
  function aggregateFragmentFromRow(row) {
19422
- const dimensionKeys = row.dimension === "model" ? { providerId: required2(row.key_1), modelId: required2(row.key_2) } : row.dimension === "kind" ? withModelKind(aiAccountingModelKind(row.key_1)) : row.dimension === "agent" ? {
20145
+ const dimensionKeys = row.dimension === "model" ? { providerId: required2(row.key_1), modelId: required2(row.key_2) } : row.dimension === "modelSeries" ? {
20146
+ providerId: required2(row.key_1),
20147
+ modelId: required2(row.key_2),
20148
+ start: new Date(required2(row.bucket_start))
20149
+ } : row.dimension === "person" ? row.key_1 && row.key_2 ? { requesterType: row.key_1, requesterId: row.key_2 } : {} : row.dimension === "kind" ? withModelKind(aiAccountingModelKind(row.key_1)) : row.dimension === "agent" ? {
19423
20150
  agentKind: required2(row.key_1),
19424
20151
  ...row.key_1 === "workflowAgent" ? { workflowId: required2(row.key_2), agentStepId: required2(row.key_3) } : {}
19425
20152
  } : row.dimension === "workflow" ? { workflowId: required2(row.key_1) } : row.dimension === "series" ? { start: new Date(required2(row.bucket_start)) } : {};
@@ -20693,11 +21420,12 @@ async function linkThread(sql, projectId, artifactId, threadId, sharedAt) {
20693
21420
  import { isIP } from "net";
20694
21421
  import { connect as connectTls } from "tls";
20695
21422
  import postgres from "postgres";
20696
- var DEFAULT_IDLE_TIMEOUT_SECONDS = 30;
21423
+ var DEFAULT_IDLE_TIMEOUT_SECONDS = 60;
20697
21424
  var DEFAULT_CONNECT_TIMEOUT_SECONDS = 10;
20698
21425
  function createPgClient(options) {
20699
21426
  const connection = {
20700
- search_path: options.schemaName
21427
+ search_path: options.schemaName,
21428
+ enable_incremental_sort: "off"
20701
21429
  };
20702
21430
  if (options.statementTimeoutMillis !== undefined) {
20703
21431
  connection.statement_timeout = options.statementTimeoutMillis;
@@ -24186,4 +24914,4 @@ export {
24186
24914
  migratePostgresStorage
24187
24915
  };
24188
24916
 
24189
- //# debugId=E730661135685C6D64756E2164756E21
24917
+ //# debugId=09BC0C242FA6748C64756E2164756E21