@sixb/pg 0.1.26 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +25 -1
  2. package/dist/agents/agent-storage.d.ts +8 -3
  3. package/dist/agents/agent-storage.d.ts.map +1 -1
  4. package/dist/agents/computers.d.ts +22 -0
  5. package/dist/agents/computers.d.ts.map +1 -0
  6. package/dist/agents/rows.d.ts +3 -0
  7. package/dist/agents/rows.d.ts.map +1 -1
  8. package/dist/agents/runs.d.ts.map +1 -1
  9. package/dist/agents/threads.d.ts +2 -1
  10. package/dist/agents/threads.d.ts.map +1 -1
  11. package/dist/agents/token-rates.d.ts +13 -0
  12. package/dist/agents/token-rates.d.ts.map +1 -0
  13. package/dist/ai-accounting-filters.d.ts +12 -0
  14. package/dist/ai-accounting-filters.d.ts.map +1 -0
  15. package/dist/ai-cost-groups.d.ts +1 -1
  16. package/dist/ai-cost-groups.d.ts.map +1 -1
  17. package/dist/index.d.ts +1 -1
  18. package/dist/index.js +893 -164
  19. package/dist/index.js.map +28 -23
  20. package/dist/migrations.d.ts.map +1 -1
  21. package/dist/objects/partitions.d.ts +10 -0
  22. package/dist/objects/partitions.d.ts.map +1 -0
  23. package/dist/objects/prepare-queries.d.ts.map +1 -1
  24. package/dist/objects/query-compiler.d.ts +5 -0
  25. package/dist/objects/query-compiler.d.ts.map +1 -1
  26. package/dist/objects/query-indexes.d.ts.map +1 -1
  27. package/dist/objects/sql-text.d.ts +3 -0
  28. package/dist/objects/sql-text.d.ts.map +1 -0
  29. package/dist/objects/text-count-index.d.ts.map +1 -1
  30. package/dist/ontology-storage/commits.d.ts.map +1 -1
  31. package/dist/ontology-storage/materialization-session.d.ts +7 -1
  32. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  33. package/dist/ontology-storage/materialization-writer.d.ts +2 -0
  34. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  35. package/dist/ontology-storage/materializations.d.ts +2 -1
  36. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  37. package/dist/pg-ai-cost-storage.d.ts.map +1 -1
  38. package/dist/pg-client.d.ts +1 -1
  39. package/dist/pg-client.d.ts.map +1 -1
  40. package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
  41. package/dist/pg-sync-run-storage.d.ts.map +1 -1
  42. package/dist/storage-errors.d.ts +4 -1
  43. package/dist/storage-errors.d.ts.map +1 -1
  44. package/package.json +3 -3
  45. package/src/agents/agent-storage.ts +10 -3
  46. package/src/agents/computers.ts +95 -0
  47. package/src/agents/rows.ts +9 -1
  48. package/src/agents/runs.ts +25 -3
  49. package/src/agents/threads.ts +111 -20
  50. package/src/agents/token-rates.ts +76 -0
  51. package/src/ai-accounting-filters.ts +48 -0
  52. package/src/ai-cost-groups.ts +37 -12
  53. package/src/index.ts +1 -1
  54. package/src/migrations/084-agent-token-rates.sql +16 -0
  55. package/src/migrations/085-agent-computer-sessions.sql +27 -0
  56. package/src/migrations/086-partition-objects.sql +93 -0
  57. package/src/migrations.ts +8 -0
  58. package/src/objects/partitions.ts +163 -0
  59. package/src/objects/prepare-queries.ts +50 -2
  60. package/src/objects/query-compiler.ts +80 -66
  61. package/src/objects/query-indexes.ts +59 -46
  62. package/src/objects/sql-text.ts +8 -0
  63. package/src/objects/text-count-index.ts +17 -12
  64. package/src/ontology-storage/commits.ts +11 -2
  65. package/src/ontology-storage/materialization-session.ts +10 -1
  66. package/src/ontology-storage/materialization-writer.ts +14 -0
  67. package/src/ontology-storage/materializations.ts +81 -7
  68. package/src/pg-ai-cost-storage.ts +119 -15
  69. package/src/pg-client.ts +9 -2
  70. package/src/pg-pipeline-run-storage.ts +5 -7
  71. package/src/pg-sync-run-storage.ts +2 -7
  72. package/src/storage-errors.ts +28 -1
@@ -28,6 +28,7 @@ import {
28
28
  type QueryAiAccountingOverviewInput,
29
29
  type SummarizeAiCostExecutionsInput,
30
30
  } from "@sixb/core/storage"
31
+ import { pgAccountingFilterValues } from "./ai-accounting-filters"
31
32
  import { listAiModelCallGroups } from "./ai-cost-groups"
32
33
  import { requestedReasoningFromRow } from "./pg-ai-usage-storage"
33
34
  import type { PgStoreClient } from "./transactions"
@@ -100,6 +101,7 @@ export class PgAiCostStorage implements AiCostStorage {
100
101
  const query = normalizeAiAccountingQuery(input)
101
102
  const providerId = query.providerId ?? null
102
103
  const modelId = query.modelId ?? null
104
+ const filter = pgAccountingFilterValues(query)
103
105
  const rows = await this.sql<AggregateRow[]>`
104
106
  WITH filtered AS (
105
107
  SELECT usage.provider_id,
@@ -130,7 +132,9 @@ export class PgAiCostStorage implements AiCostStorage {
130
132
  ) AS attribution_workflow_id,
131
133
  cost.status AS cost_status,
132
134
  cost.currency AS cost_currency,
133
- cost.amount_nanos AS cost_amount_nanos
135
+ cost.amount_nanos AS cost_amount_nanos,
136
+ execution.requested_by_user_id AS requester_user_id,
137
+ execution.requested_by_service_account_id AS requester_service_account_id
134
138
  FROM ai_model_call_usage AS usage
135
139
  JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
136
140
  LEFT JOIN agent_runs AS direct_agent
@@ -148,6 +152,26 @@ export class PgAiCostStorage implements AiCostStorage {
148
152
  AND usage.occurred_at >= ${query.from} AND usage.occurred_at < ${query.to}
149
153
  AND (${providerId}::text IS NULL OR usage.provider_id = ${providerId})
150
154
  AND (${modelId}::text IS NULL OR usage.requested_model_id = ${modelId})
155
+ AND (${filter.kind}::text IS NULL OR usage.model_kind = ${filter.kind})
156
+ AND (${filter.source}::text IS NULL OR (CASE
157
+ WHEN direct_agent.id IS NOT NULL THEN 'agent'
158
+ WHEN COALESCE(workflow_node.workflow_id,
159
+ CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
160
+ ) IS NOT NULL THEN 'workflow'
161
+ WHEN execution.executor_kind = 'action' THEN 'action'
162
+ ELSE 'other'
163
+ END) = ${filter.source})
164
+ AND (${filter.workflow}::text IS NULL OR COALESCE(workflow_node.workflow_id,
165
+ CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
166
+ ) = ${filter.workflow})
167
+ AND (
168
+ ${filter.mode}::text IS NULL
169
+ OR (${filter.mode} = 'none' AND execution.requested_by_user_id IS NULL
170
+ AND execution.requested_by_service_account_id IS NULL)
171
+ OR (${filter.mode} = 'user' AND execution.requested_by_user_id = ${filter.id})
172
+ OR (${filter.mode} = 'serviceAccount'
173
+ AND execution.requested_by_service_account_id = ${filter.id})
174
+ )
151
175
  ), classified AS (
152
176
  SELECT filtered.*,
153
177
  CASE
@@ -178,8 +202,27 @@ export class PgAiCostStorage implements AiCostStorage {
178
202
  classified.*
179
203
  FROM classified
180
204
  UNION ALL
205
+ SELECT 'modelSeries', provider_id, requested_model_id, NULL,
206
+ CASE ${input.bucket}::text
207
+ WHEN 'hour' THEN
208
+ date_trunc('hour', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
209
+ WHEN 'day' THEN
210
+ date_trunc('day', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
211
+ ELSE date_trunc('week', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
212
+ END,
213
+ classified.*
214
+ FROM classified
215
+ UNION ALL
181
216
  SELECT 'model', provider_id, requested_model_id, NULL, NULL, classified.* FROM classified
182
217
  UNION ALL
218
+ SELECT 'person',
219
+ CASE
220
+ WHEN requester_user_id IS NOT NULL THEN 'user'
221
+ WHEN requester_service_account_id IS NOT NULL THEN 'serviceAccount'
222
+ END,
223
+ COALESCE(requester_user_id, requester_service_account_id), NULL, NULL, classified.*
224
+ FROM classified
225
+ UNION ALL
183
226
  SELECT 'kind', model_kind, NULL, NULL, NULL, classified.* FROM classified
184
227
  UNION ALL
185
228
  SELECT 'agent', attribution_agent_kind, attribution_workflow_id, attribution_agent_step_id, NULL, classified.*
@@ -243,9 +286,20 @@ export class PgAiCostStorage implements AiCostStorage {
243
286
  const modelId = query.modelId ?? null
244
287
  const executionId = query.executionId ?? null
245
288
  const valuationStatus = query.valuationStatus ?? null
289
+ const filter = pgAccountingFilterValues(query)
246
290
  const [count] = await this.sql<CountRow[]>`
247
291
  SELECT COUNT(*)::text AS total
248
292
  FROM ai_model_call_usage AS usage
293
+ JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
294
+ LEFT JOIN agent_runs AS direct_agent
295
+ ON direct_agent.project_id = usage.project_id
296
+ AND direct_agent.execution_id = usage.execution_id
297
+ LEFT JOIN workflow_agent_node_runs AS workflow_agent
298
+ ON workflow_agent.project_id = usage.project_id
299
+ AND workflow_agent.execution_id = usage.execution_id
300
+ LEFT JOIN workflow_node_runs AS workflow_node
301
+ ON workflow_node.project_id = workflow_agent.project_id
302
+ AND workflow_node.id = workflow_agent.node_run_id
249
303
  LEFT JOIN ai_model_call_valuations AS cost
250
304
  ON cost.project_id = usage.project_id AND cost.usage_record_id = usage.id
251
305
  WHERE usage.project_id = ${query.projectId}
@@ -260,6 +314,26 @@ export class PgAiCostStorage implements AiCostStorage {
260
314
  ELSE 'unvalued'
261
315
  END
262
316
  )
317
+ AND (${filter.kind}::text IS NULL OR usage.model_kind = ${filter.kind})
318
+ AND (${filter.source}::text IS NULL OR (CASE
319
+ WHEN direct_agent.id IS NOT NULL THEN 'agent'
320
+ WHEN COALESCE(workflow_node.workflow_id,
321
+ CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
322
+ ) IS NOT NULL THEN 'workflow'
323
+ WHEN execution.executor_kind = 'action' THEN 'action'
324
+ ELSE 'other'
325
+ END) = ${filter.source})
326
+ AND (${filter.workflow}::text IS NULL OR COALESCE(workflow_node.workflow_id,
327
+ CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
328
+ ) = ${filter.workflow})
329
+ AND (
330
+ ${filter.mode}::text IS NULL
331
+ OR (${filter.mode} = 'none' AND execution.requested_by_user_id IS NULL
332
+ AND execution.requested_by_service_account_id IS NULL)
333
+ OR (${filter.mode} = 'user' AND execution.requested_by_user_id = ${filter.id})
334
+ OR (${filter.mode} = 'serviceAccount'
335
+ AND execution.requested_by_service_account_id = ${filter.id})
336
+ )
263
337
  `
264
338
  const rows = await this.sql<JoinedRow[]>`
265
339
  SELECT usage.*,
@@ -315,6 +389,26 @@ export class PgAiCostStorage implements AiCostStorage {
315
389
  ELSE 'unvalued'
316
390
  END
317
391
  )
392
+ AND (${filter.kind}::text IS NULL OR usage.model_kind = ${filter.kind})
393
+ AND (${filter.source}::text IS NULL OR (CASE
394
+ WHEN direct_agent.id IS NOT NULL THEN 'agent'
395
+ WHEN COALESCE(workflow_node.workflow_id,
396
+ CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
397
+ ) IS NOT NULL THEN 'workflow'
398
+ WHEN execution.executor_kind = 'action' THEN 'action'
399
+ ELSE 'other'
400
+ END) = ${filter.source})
401
+ AND (${filter.workflow}::text IS NULL OR COALESCE(workflow_node.workflow_id,
402
+ CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
403
+ ) = ${filter.workflow})
404
+ AND (
405
+ ${filter.mode}::text IS NULL
406
+ OR (${filter.mode} = 'none' AND execution.requested_by_user_id IS NULL
407
+ AND execution.requested_by_service_account_id IS NULL)
408
+ OR (${filter.mode} = 'user' AND execution.requested_by_user_id = ${filter.id})
409
+ OR (${filter.mode} = 'serviceAccount'
410
+ AND execution.requested_by_service_account_id = ${filter.id})
411
+ )
318
412
  ORDER BY usage.occurred_at DESC, usage.id ASC
319
413
  LIMIT ${query.limit} OFFSET ${query.offset}
320
414
  `
@@ -475,20 +569,30 @@ function aggregateFragmentFromRow(row: AggregateRow): AiAccountingAggregateFragm
475
569
  const dimensionKeys =
476
570
  row.dimension === "model"
477
571
  ? { providerId: required(row.key_1), modelId: required(row.key_2) }
478
- : row.dimension === "kind"
479
- ? withModelKind(aiAccountingModelKind(row.key_1))
480
- : row.dimension === "agent"
481
- ? {
482
- agentKind: required(row.key_1) as "agent" | "workflowAgent",
483
- ...(row.key_1 === "workflowAgent"
484
- ? { workflowId: required(row.key_2), agentStepId: required(row.key_3) }
485
- : {}),
486
- }
487
- : row.dimension === "workflow"
488
- ? { workflowId: required(row.key_1) }
489
- : row.dimension === "series"
490
- ? { start: new Date(required(row.bucket_start)) }
491
- : {}
572
+ : row.dimension === "modelSeries"
573
+ ? {
574
+ providerId: required(row.key_1),
575
+ modelId: required(row.key_2),
576
+ start: new Date(required(row.bucket_start)),
577
+ }
578
+ : row.dimension === "person"
579
+ ? row.key_1 && row.key_2
580
+ ? { requesterType: row.key_1 as "user" | "serviceAccount", requesterId: row.key_2 }
581
+ : {}
582
+ : row.dimension === "kind"
583
+ ? withModelKind(aiAccountingModelKind(row.key_1))
584
+ : row.dimension === "agent"
585
+ ? {
586
+ agentKind: required(row.key_1) as "agent" | "workflowAgent",
587
+ ...(row.key_1 === "workflowAgent"
588
+ ? { workflowId: required(row.key_2), agentStepId: required(row.key_3) }
589
+ : {}),
590
+ }
591
+ : row.dimension === "workflow"
592
+ ? { workflowId: required(row.key_1) }
593
+ : row.dimension === "series"
594
+ ? { start: new Date(required(row.bucket_start)) }
595
+ : {}
492
596
  return {
493
597
  dimension: row.dimension,
494
598
  ...dimensionKeys,
package/src/pg-client.ts CHANGED
@@ -54,7 +54,7 @@ export interface CreatePgClientOptions {
54
54
  readonly max: number
55
55
  /** Schema pinned via `search_path` on every connection. */
56
56
  readonly schemaName: string
57
- /** Close idle connections after this many ms (frees slots back to the server). Default 30s. */
57
+ /** Close idle connections after this many ms (frees slots back to the server). Default 60s. */
58
58
  readonly idleTimeoutMillis?: number
59
59
  /** Per-connection `statement_timeout` (ms). Unset = no timeout. */
60
60
  readonly statementTimeoutMillis?: number
@@ -73,7 +73,10 @@ export interface CreatePgClientOptions {
73
73
  readonly ssl?: boolean | "require" | "prefer"
74
74
  }
75
75
 
76
- const DEFAULT_IDLE_TIMEOUT_SECONDS = 30
76
+ // A new connection has an empty catalog cache, so its first query plans several times slower. A
77
+ // short pause should not cost that. Not much longer than this: idle connections hold their slots,
78
+ // and one a failover dropped silently can hang the next query until TCP gives up.
79
+ const DEFAULT_IDLE_TIMEOUT_SECONDS = 60
77
80
  const DEFAULT_CONNECT_TIMEOUT_SECONDS = 10
78
81
 
79
82
  /** Build a configured porsager `postgres` pool. */
@@ -81,6 +84,10 @@ export function createPgClient(options: CreatePgClientOptions): SQL {
81
84
  // GUCs applied to every connection at startup.
82
85
  const connection: Record<string, string | number | boolean> = {
83
86
  search_path: options.schemaName,
87
+ // PostgreSQL costs an incremental sort as if its presorted key had many values. Over a sort
88
+ // index the first key is a null rank with one value, so a query with a sparse filter reads
89
+ // every row of the type where the planner expected a few.
90
+ enable_incremental_sort: "off",
84
91
  }
85
92
  if (options.statementTimeoutMillis !== undefined) {
86
93
  connection.statement_timeout = options.statementTimeoutMillis
@@ -1,7 +1,10 @@
1
1
  import type { JsonValue } from "@sixb/core"
2
2
  import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
3
3
  import { assertPipelineRunExecution } from "@sixb/core/internal/pipeline-run-storage-provider"
4
- import { canRequeuePipelineRunAfterEnqueueFailure } from "@sixb/core/internal/storage"
4
+ import {
5
+ canFinishPipelineRun,
6
+ canRequeuePipelineRunAfterEnqueueFailure,
7
+ } from "@sixb/core/internal/storage"
5
8
  import type { DatasetVersionRef } from "@sixb/core/lake-storage"
6
9
  import type {
7
10
  ExecutionStorage,
@@ -137,12 +140,7 @@ export class PgPipelineRunStorage implements PipelineRunStorage {
137
140
  )
138
141
  }
139
142
 
140
- const enqueueFailure =
141
- input.status === "failed" && input.error.code === "queue.enqueue_failed"
142
- if (
143
- (enqueueFailure && existing.status !== "queued") ||
144
- (!enqueueFailure && existing.status !== "running")
145
- ) {
143
+ if (!canFinishPipelineRun(existing.status, input)) {
146
144
  throw new PipelineRunError(
147
145
  `[SixbPg] Pipeline run '${input.id}' cannot finish from status '${existing.status}'.`
148
146
  )
@@ -1,6 +1,6 @@
1
1
  import type { JsonValue } from "@sixb/core"
2
2
  import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
3
- import { canRequeueSyncRunAfterEnqueueFailure } from "@sixb/core/internal/storage"
3
+ import { canFinishSyncRun, canRequeueSyncRunAfterEnqueueFailure } from "@sixb/core/internal/storage"
4
4
  import { assertSyncRunExecution } from "@sixb/core/internal/sync-run-storage-provider"
5
5
  import type {
6
6
  ExecutionStorage,
@@ -134,12 +134,7 @@ export class PgSyncRunStorage implements SyncRunStorage {
134
134
  )
135
135
  }
136
136
 
137
- const enqueueFailure =
138
- input.status === "failed" && input.error.code === "queue.enqueue_failed"
139
- if (
140
- (enqueueFailure && existing.status !== "queued") ||
141
- (!enqueueFailure && existing.status !== "running")
142
- ) {
137
+ if (!canFinishSyncRun(existing.status, input)) {
143
138
  throw new SyncRunError(
144
139
  `[SixbPg] Sync run '${input.id}' cannot finish from status '${existing.status}'.`
145
140
  )
@@ -65,6 +65,9 @@ const LOCK_TABLES = [
65
65
 
66
66
  const OUT_OF_MEMORY = "53200"
67
67
 
68
+ /** PostgreSQL SQLSTATE for a statement sent after an earlier one aborted its transaction. */
69
+ const IN_FAILED_SQL_TRANSACTION = "25P02"
70
+
68
71
  /** The SQLSTATE / system error code carried by a thrown database or connection error. */
69
72
  export function pgErrorCode(error: unknown): string | undefined {
70
73
  if (error instanceof Error) {
@@ -103,6 +106,12 @@ export function isRetryableTransactionConflict(error: unknown): boolean {
103
106
  return code !== undefined && RETRYABLE_TRANSACTION_CONFLICT_CODES.has(code)
104
107
  }
105
108
 
109
+ /** Another session held a lock this one gave up on, or PostgreSQL broke a lock cycle. */
110
+ export function isLockConflict(error: unknown): boolean {
111
+ const code = pgErrorCode(error)
112
+ return code === "55P03" || code === "40P01"
113
+ }
114
+
106
115
  /**
107
116
  * True when the error means the session is gone: PostgreSQL has already rolled back its open
108
117
  * transaction and released its locks, and nothing more may be sent on that connection.
@@ -124,12 +133,16 @@ export function isConnectionLost(error: unknown): boolean {
124
133
  /**
125
134
  * Replace an error that says something about the database rather than the statement: a lost or
126
135
  * unopenable connection becomes the retryable `storage.unavailable`, a full lock table an error
127
- * naming the table and the setting that sizes it. Every other error is returned unchanged.
136
+ * naming the table and the setting that sizes it, and 25P02 an error saying the first failure was
137
+ * lost. Every other error is returned unchanged.
128
138
  */
129
139
  export function explainPgError(error: unknown): unknown {
130
140
  if (isStorageUnavailable(error)) {
131
141
  return storageUnavailable(error)
132
142
  }
143
+ if (pgErrorCode(error) === IN_FAILED_SQL_TRANSACTION) {
144
+ return failedStatementLost(error)
145
+ }
133
146
  if (!(error instanceof Error) || pgErrorCode(error) !== OUT_OF_MEMORY) {
134
147
  return error
135
148
  }
@@ -150,6 +163,20 @@ export function explainPgError(error: unknown): unknown {
150
163
  )
151
164
  }
152
165
 
166
+ /**
167
+ * PostgreSQL rejects every statement after one that failed, without saying which one failed or
168
+ * why. The transaction's own error, a retryable serialization failure included, was replaced by a
169
+ * statement that the code handling it sent: a cleanup, a log write, a fallback query.
170
+ */
171
+ function failedStatementLost(error: unknown): Error {
172
+ return new Error(
173
+ "[SixbPg] A statement was sent after an earlier statement of the same transaction failed, so " +
174
+ "PostgreSQL rejected it (SQLSTATE 25P02) and the earlier error was lost. Code that handles " +
175
+ "a failure inside a transaction must not send more SQL on it: let the error propagate.",
176
+ { cause: error }
177
+ )
178
+ }
179
+
153
180
  function isStorageUnavailable(error: unknown): boolean {
154
181
  const code = pgErrorCode(error)
155
182
  return isConnectionLost(error) || (code !== undefined && CONNECT_FAILED_CODES.has(code))