@sixb/pg 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/dist/agents/rows.d.ts +3 -1
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/threads.d.ts +3 -2
- package/dist/agents/threads.d.ts.map +1 -1
- package/dist/auth-storage/access-tokens.d.ts.map +1 -1
- package/dist/auth-storage/device-authorizations.d.ts.map +1 -1
- package/dist/auth-storage/rows.d.ts +6 -3
- package/dist/auth-storage/rows.d.ts.map +1 -1
- package/dist/auth-storage/sessions.d.ts +8 -1
- package/dist/auth-storage/sessions.d.ts.map +1 -1
- package/dist/auth-storage/shared.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +917 -122
- package/dist/index.js.map +26 -23
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +8 -1
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts +1 -0
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +1 -0
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/rows.d.ts +1 -0
- package/dist/objects/rows.d.ts.map +1 -1
- package/dist/objects/storage.d.ts +1 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/ontology-storage/index.d.ts +4 -0
- package/dist/ontology-storage/index.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +1 -2
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +2 -1
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/vector-indexing.d.ts +20 -0
- package/dist/ontology-storage/vector-indexing.d.ts.map +1 -0
- package/dist/ontology-storage/vectors.d.ts +16 -0
- package/dist/ontology-storage/vectors.d.ts.map +1 -0
- package/dist/pg-ai-cost-storage.d.ts.map +1 -1
- package/dist/pg-file-upload-session-storage.d.ts +6 -0
- package/dist/pg-file-upload-session-storage.d.ts.map +1 -0
- package/dist/pg-workflow-intervention-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/rows.ts +21 -0
- package/src/agents/threads.ts +47 -2
- package/src/auth-storage/access-tokens.ts +10 -14
- package/src/auth-storage/auth-storage.ts +4 -6
- package/src/auth-storage/device-authorizations.ts +3 -4
- package/src/auth-storage/rows.ts +17 -7
- package/src/auth-storage/sessions.ts +82 -4
- package/src/auth-storage/shared.ts +8 -2
- package/src/index.ts +5 -0
- package/src/migrations/041-agent-thread-workspaces.sql +2 -0
- package/src/migrations/042-vector-profiles.sql +28 -0
- package/src/migrations/043-vector-indexing.sql +24 -0
- package/src/migrations/044-vector-batching.sql +3 -0
- package/src/migrations/045-agent-thread-sandbox-state.sql +2 -0
- package/src/migrations/046-workflow-intervention-principals.sql +15 -0
- package/src/migrations/047-ontology-commit-attribution.sql +50 -0
- package/src/migrations/048-file-upload-sessions.sql +34 -0
- package/src/migrations/049-native-sessions.sql +19 -0
- package/src/migrations.ts +26 -0
- package/src/objects/query-compiler.ts +102 -11
- package/src/objects/read-scope.ts +26 -25
- package/src/objects/reader.ts +33 -1
- package/src/objects/rows.ts +5 -1
- package/src/objects/storage.ts +22 -2
- package/src/ontology-storage/index.ts +15 -0
- package/src/ontology-storage/materializations.ts +28 -58
- package/src/ontology-storage/shared.ts +7 -3
- package/src/ontology-storage/vector-indexing.ts +168 -0
- package/src/ontology-storage/vectors.ts +172 -0
- package/src/pg-ai-cost-storage.ts +4 -0
- package/src/pg-file-upload-session-storage.ts +143 -0
- package/src/pg-workflow-intervention-storage.ts +11 -15
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { parseSixbFailure } from "@sixb/core/internal/errors"
|
|
2
|
+
import {
|
|
3
|
+
type MaterializationSession,
|
|
4
|
+
type OntologyVectorIndexingStorage,
|
|
5
|
+
VECTOR_INDEXING_FAILURE_CODES,
|
|
6
|
+
type VectorIndexingRequest,
|
|
7
|
+
type VectorIndexingWork,
|
|
8
|
+
} from "@sixb/core/storage"
|
|
9
|
+
import type { SQLClient } from "../pg-client"
|
|
10
|
+
import { jsonParameter as json, type PgRootOperation } from "./shared"
|
|
11
|
+
|
|
12
|
+
interface WorkRow {
|
|
13
|
+
batch_id: string | null
|
|
14
|
+
request: VectorIndexingRequest
|
|
15
|
+
status: VectorIndexingWork["status"]
|
|
16
|
+
available_at: Date | string
|
|
17
|
+
values_json: number[] | null
|
|
18
|
+
error: string | null
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export class PgVectorIndexingStorage implements OntologyVectorIndexingStorage {
|
|
22
|
+
constructor(
|
|
23
|
+
private readonly sql: SQLClient,
|
|
24
|
+
private readonly run: PgRootOperation,
|
|
25
|
+
private readonly assertSession: (session: MaterializationSession, projectId: string) => void
|
|
26
|
+
) {}
|
|
27
|
+
|
|
28
|
+
async schedule(input: Parameters<OntologyVectorIndexingStorage["schedule"]>[0]) {
|
|
29
|
+
this.assertSession(input.session, input.projectId)
|
|
30
|
+
if (input.deleted.length)
|
|
31
|
+
await this.sql`
|
|
32
|
+
DELETE FROM object_vector_indexing AS work
|
|
33
|
+
USING jsonb_array_elements(${json(this.sql, input.deleted)}::jsonb) AS deleted(ref)
|
|
34
|
+
WHERE work.project_id = ${input.projectId}
|
|
35
|
+
AND work.object_type_id = deleted.ref->>'objectTypeId'
|
|
36
|
+
AND work.primary_id = deleted.ref->>'primaryId'
|
|
37
|
+
`
|
|
38
|
+
if (input.requests.length)
|
|
39
|
+
await this.sql`
|
|
40
|
+
INSERT INTO object_vector_indexing (
|
|
41
|
+
project_id, id, object_type_id, primary_id, profile, batch_id, request, status, available_at, dispatch_at
|
|
42
|
+
)
|
|
43
|
+
SELECT ${input.projectId}, entry->>'id', entry#>>'{ref,objectTypeId}',
|
|
44
|
+
entry#>>'{ref,primaryId}', entry->>'profile', entry->>'batchId', entry - 'batchId', 'pending',
|
|
45
|
+
${input.availableAt}::timestamptz, ${input.availableAt}::timestamptz
|
|
46
|
+
FROM jsonb_array_elements(${json(this.sql, input.requests)}::jsonb) AS pending(entry)
|
|
47
|
+
ON CONFLICT (project_id, object_type_id, primary_id, profile) DO UPDATE SET
|
|
48
|
+
id = EXCLUDED.id, batch_id = EXCLUDED.batch_id, request = EXCLUDED.request, status = 'pending',
|
|
49
|
+
available_at = EXCLUDED.available_at, dispatch_at = EXCLUDED.dispatch_at,
|
|
50
|
+
values_json = NULL, error = NULL
|
|
51
|
+
`
|
|
52
|
+
}
|
|
53
|
+
async complete(input: Parameters<OntologyVectorIndexingStorage["complete"]>[0]) {
|
|
54
|
+
this.assertSession(input.session, input.projectId)
|
|
55
|
+
if (!input.entries.length) return
|
|
56
|
+
await this.sql`
|
|
57
|
+
DELETE FROM object_vector_indexing AS work
|
|
58
|
+
USING jsonb_array_elements(${json(this.sql, input.entries)}::jsonb) AS completed(entry)
|
|
59
|
+
WHERE work.project_id = ${input.projectId}
|
|
60
|
+
AND work.object_type_id = entry#>>'{ref,objectTypeId}'
|
|
61
|
+
AND work.primary_id = entry#>>'{ref,primaryId}'
|
|
62
|
+
AND work.profile = entry->>'profile'
|
|
63
|
+
AND work.request->>'configuration' = entry->>'configuration'
|
|
64
|
+
AND work.request->>'sourceFingerprint' = entry->>'sourceFingerprint'
|
|
65
|
+
`
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async dispatched(input: Parameters<OntologyVectorIndexingStorage["dispatched"]>[0]) {
|
|
69
|
+
if (!input.ids.length) return
|
|
70
|
+
await this.run(async (sql) => {
|
|
71
|
+
await sql`UPDATE object_vector_indexing SET dispatch_at = ${input.nextDispatchAt} WHERE project_id = ${input.projectId} AND id = ANY(${sql.array([...input.ids])}::text[])`
|
|
72
|
+
})
|
|
73
|
+
}
|
|
74
|
+
async get(input: Parameters<OntologyVectorIndexingStorage["get"]>[0]) {
|
|
75
|
+
return this.run(async (sql) => {
|
|
76
|
+
const rows = await sql<
|
|
77
|
+
WorkRow[]
|
|
78
|
+
>`SELECT batch_id, request, status, available_at, values_json, error FROM object_vector_indexing WHERE project_id = ${input.projectId} AND id = ${input.id}`
|
|
79
|
+
return rows[0] ? decode(rows[0]) : null
|
|
80
|
+
})
|
|
81
|
+
}
|
|
82
|
+
async getBatch(input: Parameters<OntologyVectorIndexingStorage["getBatch"]>[0]) {
|
|
83
|
+
return this.run(async (sql) =>
|
|
84
|
+
(
|
|
85
|
+
await sql<WorkRow[]>`
|
|
86
|
+
SELECT batch_id, request, status, available_at, values_json, error
|
|
87
|
+
FROM object_vector_indexing WHERE project_id = ${input.projectId}
|
|
88
|
+
AND batch_id = ${input.batchId} ORDER BY id`
|
|
89
|
+
).map(decode)
|
|
90
|
+
)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async getBatchMember(input: Parameters<OntologyVectorIndexingStorage["getBatchMember"]>[0]) {
|
|
94
|
+
return this.run(async (sql) => {
|
|
95
|
+
const rows = await sql<WorkRow[]>`
|
|
96
|
+
SELECT batch_id, request, status, available_at, values_json, error
|
|
97
|
+
FROM object_vector_indexing WHERE project_id = ${input.projectId}
|
|
98
|
+
AND object_type_id = ${input.ref.objectTypeId} AND primary_id = ${input.ref.primaryId}
|
|
99
|
+
AND profile = ${input.profile} AND batch_id = ${input.batchId}`
|
|
100
|
+
return rows[0] ? decode(rows[0]) : null
|
|
101
|
+
})
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async updateBatch(input: Parameters<OntologyVectorIndexingStorage["updateBatch"]>[0]) {
|
|
105
|
+
if (!input.updates.length) return true
|
|
106
|
+
return this.run(async (sql) => {
|
|
107
|
+
const rows = await sql<{ id: string }[]>`
|
|
108
|
+
WITH changes AS MATERIALIZED (
|
|
109
|
+
SELECT item FROM jsonb_array_elements(${json(sql, input.updates)}::jsonb) AS entries(item)
|
|
110
|
+
), locked AS MATERIALIZED (
|
|
111
|
+
SELECT work.id FROM object_vector_indexing AS work
|
|
112
|
+
JOIN changes ON work.id = item->>'id' AND work.status = item->>'expectedStatus'
|
|
113
|
+
WHERE work.project_id = ${input.projectId} ORDER BY work.id FOR UPDATE OF work
|
|
114
|
+
)
|
|
115
|
+
UPDATE object_vector_indexing AS work SET
|
|
116
|
+
status = changes.item->>'status', available_at = (changes.item->>'availableAt')::timestamptz,
|
|
117
|
+
values_json = COALESCE(changes.item->'values', work.values_json),
|
|
118
|
+
error = (changes.item->'error')::text
|
|
119
|
+
FROM changes, locked
|
|
120
|
+
WHERE work.project_id = ${input.projectId} AND work.id = locked.id
|
|
121
|
+
AND work.id = changes.item->>'id'
|
|
122
|
+
AND (${!input.requireAll} OR (SELECT count(*) FROM locked) = ${input.updates.length})
|
|
123
|
+
RETURNING work.id`
|
|
124
|
+
return !input.requireAll || rows.length === input.updates.length
|
|
125
|
+
})
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async listDue(input: Parameters<OntologyVectorIndexingStorage["listDue"]>[0]) {
|
|
129
|
+
return this.run(async (sql) =>
|
|
130
|
+
(
|
|
131
|
+
await sql<
|
|
132
|
+
WorkRow[]
|
|
133
|
+
>`SELECT batch_id, request, status, available_at, NULL AS values_json, error FROM object_vector_indexing WHERE project_id = ${input.projectId} AND status <> 'failed' AND dispatch_at <= ${input.now} ORDER BY dispatch_at, id LIMIT ${input.limit}`
|
|
134
|
+
).map(decode)
|
|
135
|
+
)
|
|
136
|
+
}
|
|
137
|
+
async update(input: Parameters<OntologyVectorIndexingStorage["update"]>[0]) {
|
|
138
|
+
return this.run(
|
|
139
|
+
async (sql) =>
|
|
140
|
+
(
|
|
141
|
+
await sql`
|
|
142
|
+
UPDATE object_vector_indexing SET status = ${input.status}, available_at = ${input.availableAt},
|
|
143
|
+
values_json = COALESCE(${input.values ? json(sql, input.values) : null}, values_json), error = ${input.error ? JSON.stringify(input.error) : null}
|
|
144
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${input.expectedStatus} RETURNING id
|
|
145
|
+
`
|
|
146
|
+
).length === 1
|
|
147
|
+
)
|
|
148
|
+
}
|
|
149
|
+
async remove(input: Parameters<OntologyVectorIndexingStorage["remove"]>[0]) {
|
|
150
|
+
await this.run(async (sql) => {
|
|
151
|
+
await sql`DELETE FROM object_vector_indexing WHERE project_id = ${input.projectId} AND id = ${input.id}`
|
|
152
|
+
})
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
function decode(row: WorkRow): VectorIndexingWork {
|
|
156
|
+
return {
|
|
157
|
+
...row.request,
|
|
158
|
+
...(row.batch_id ? { batchId: row.batch_id } : {}),
|
|
159
|
+
status: row.status,
|
|
160
|
+
availableAt: new Date(row.available_at).toISOString(),
|
|
161
|
+
...(row.values_json ? { values: row.values_json } : {}),
|
|
162
|
+
...(row.error
|
|
163
|
+
? {
|
|
164
|
+
error: parseSixbFailure(row.error, VECTOR_INDEXING_FAILURE_CODES),
|
|
165
|
+
}
|
|
166
|
+
: {}),
|
|
167
|
+
}
|
|
168
|
+
}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { MaterializationConflictError } from "@sixb/core/internal/materialization"
|
|
2
|
+
import type {
|
|
3
|
+
MaterializationSession,
|
|
4
|
+
ObjectVectorState,
|
|
5
|
+
OntologyVectorStorage,
|
|
6
|
+
} from "@sixb/core/storage"
|
|
7
|
+
import type { SQLClient } from "../pg-client"
|
|
8
|
+
import { jsonParameter as json, type PgRootOperation } from "./shared"
|
|
9
|
+
|
|
10
|
+
interface VectorMetadataRow {
|
|
11
|
+
readonly profile: string
|
|
12
|
+
readonly configuration: string
|
|
13
|
+
readonly source: string[]
|
|
14
|
+
readonly source_fingerprint: string
|
|
15
|
+
readonly last_commit_id: string
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export class PgOntologyVectorStorage implements OntologyVectorStorage {
|
|
19
|
+
constructor(
|
|
20
|
+
private readonly sql: SQLClient,
|
|
21
|
+
private readonly runRootOperation: PgRootOperation,
|
|
22
|
+
private readonly assertSession: (
|
|
23
|
+
session: MaterializationSession,
|
|
24
|
+
projectId: string,
|
|
25
|
+
commitId?: string
|
|
26
|
+
) => void
|
|
27
|
+
) {}
|
|
28
|
+
|
|
29
|
+
async list(
|
|
30
|
+
input: Parameters<OntologyVectorStorage["list"]>[0]
|
|
31
|
+
): Promise<readonly ObjectVectorState[]> {
|
|
32
|
+
return this.runRootOperation(async (sql) => {
|
|
33
|
+
const rows = await sql<VectorMetadataRow[]>`
|
|
34
|
+
SELECT profile, configuration, source, source_fingerprint, last_commit_id
|
|
35
|
+
FROM object_vectors
|
|
36
|
+
WHERE project_id = ${input.projectId}
|
|
37
|
+
AND object_type_id = ${input.ref.objectTypeId} AND primary_id = ${input.ref.primaryId}
|
|
38
|
+
ORDER BY profile
|
|
39
|
+
`
|
|
40
|
+
return rows.map((row) => ({
|
|
41
|
+
ref: { ...input.ref },
|
|
42
|
+
profile: row.profile,
|
|
43
|
+
configuration: row.configuration,
|
|
44
|
+
source: row.source,
|
|
45
|
+
sourceFingerprint: row.source_fingerprint,
|
|
46
|
+
lastCommitId: row.last_commit_id,
|
|
47
|
+
}))
|
|
48
|
+
})
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async listBatch(
|
|
52
|
+
input: Parameters<OntologyVectorStorage["listBatch"]>[0]
|
|
53
|
+
): Promise<readonly ObjectVectorState[]> {
|
|
54
|
+
if (!input.refs.length) return []
|
|
55
|
+
return this.runRootOperation(async (sql) => {
|
|
56
|
+
const rows = await sql<
|
|
57
|
+
(VectorMetadataRow & { object_type_id: string; primary_id: string })[]
|
|
58
|
+
>`
|
|
59
|
+
SELECT stored.object_type_id, stored.primary_id, stored.profile, stored.configuration,
|
|
60
|
+
stored.source, stored.source_fingerprint, stored.last_commit_id
|
|
61
|
+
FROM (SELECT DISTINCT ref->>'objectTypeId' AS object_type_id, ref->>'primaryId' AS primary_id
|
|
62
|
+
FROM jsonb_array_elements(${json(sql, input.refs)}::jsonb) AS requested(ref)) AS requested
|
|
63
|
+
JOIN object_vectors AS stored ON stored.project_id = ${input.projectId}
|
|
64
|
+
AND stored.object_type_id = requested.object_type_id AND stored.primary_id = requested.primary_id
|
|
65
|
+
ORDER BY stored.object_type_id, stored.primary_id, stored.profile
|
|
66
|
+
`
|
|
67
|
+
return rows.map((row) => ({
|
|
68
|
+
ref: { objectTypeId: row.object_type_id, primaryId: row.primary_id },
|
|
69
|
+
profile: row.profile,
|
|
70
|
+
configuration: row.configuration,
|
|
71
|
+
source: row.source,
|
|
72
|
+
sourceFingerprint: row.source_fingerprint,
|
|
73
|
+
lastCommitId: row.last_commit_id,
|
|
74
|
+
}))
|
|
75
|
+
})
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async removeBatch(input: Parameters<OntologyVectorStorage["removeBatch"]>[0]): Promise<void> {
|
|
79
|
+
this.assertSession(input.session, input.projectId)
|
|
80
|
+
if (!input.entries.length) return
|
|
81
|
+
const removed = await this.sql`
|
|
82
|
+
DELETE FROM object_vectors AS stored
|
|
83
|
+
USING jsonb_array_elements(${json(this.sql, input.entries)}::jsonb) AS requested(entry)
|
|
84
|
+
WHERE stored.project_id = ${input.projectId}
|
|
85
|
+
AND stored.object_type_id = entry#>>'{ref,objectTypeId}'
|
|
86
|
+
AND stored.primary_id = entry#>>'{ref,primaryId}' AND stored.profile = entry->>'profile'
|
|
87
|
+
AND stored.last_commit_id = entry->>'expectedCommitId'
|
|
88
|
+
RETURNING stored.profile
|
|
89
|
+
`
|
|
90
|
+
if (removed.length !== input.entries.length) throw vectorConflict()
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async write(input: Parameters<OntologyVectorStorage["write"]>[0]): Promise<void> {
|
|
94
|
+
await this.writeBatch({ session: input.session, projectId: input.projectId, entries: [input] })
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async writeBatch(input: Parameters<OntologyVectorStorage["writeBatch"]>[0]): Promise<void> {
|
|
98
|
+
const entries = input.entries.map(({ value, expectedCommitId }) => {
|
|
99
|
+
this.assertSession(input.session, input.projectId, value.lastCommitId)
|
|
100
|
+
return {
|
|
101
|
+
object_type_id: value.ref.objectTypeId,
|
|
102
|
+
primary_id: value.ref.primaryId,
|
|
103
|
+
profile: value.profile,
|
|
104
|
+
configuration: value.configuration,
|
|
105
|
+
source: [...value.source],
|
|
106
|
+
source_fingerprint: value.sourceFingerprint,
|
|
107
|
+
embedding: [...value.values],
|
|
108
|
+
last_commit_id: value.lastCommitId,
|
|
109
|
+
expected_commit_id: expectedCommitId,
|
|
110
|
+
}
|
|
111
|
+
})
|
|
112
|
+
const inserts = entries.filter((entry) => entry.expected_commit_id === null)
|
|
113
|
+
const updates = entries.filter((entry) => entry.expected_commit_id !== null)
|
|
114
|
+
|
|
115
|
+
// Keep insert-if-absent and revision-checked update distinct: an upsert could recreate
|
|
116
|
+
// a deleted vector or overwrite a representation produced after preparation.
|
|
117
|
+
if (inserts.length) {
|
|
118
|
+
const rows = await this.sql`
|
|
119
|
+
INSERT INTO object_vectors (
|
|
120
|
+
project_id, object_type_id, primary_id, profile, configuration,
|
|
121
|
+
source, source_fingerprint, embedding, last_commit_id
|
|
122
|
+
)
|
|
123
|
+
SELECT ${input.projectId}, object_type_id, primary_id, profile, configuration,
|
|
124
|
+
source, source_fingerprint, embedding, last_commit_id
|
|
125
|
+
FROM jsonb_to_recordset(${json(this.sql, inserts)}::jsonb) AS incoming(
|
|
126
|
+
object_type_id text, primary_id text, profile text, configuration text,
|
|
127
|
+
source text[], source_fingerprint text, embedding real[], last_commit_id text
|
|
128
|
+
)
|
|
129
|
+
ORDER BY object_type_id, primary_id, profile
|
|
130
|
+
ON CONFLICT (project_id, object_type_id, primary_id, profile) DO NOTHING
|
|
131
|
+
RETURNING profile`
|
|
132
|
+
if (rows.length !== inserts.length) throw vectorConflict()
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
if (updates.length) {
|
|
136
|
+
const rows = await this.sql`
|
|
137
|
+
UPDATE object_vectors AS stored SET
|
|
138
|
+
configuration = incoming.configuration, source = incoming.source,
|
|
139
|
+
source_fingerprint = incoming.source_fingerprint, embedding = incoming.embedding,
|
|
140
|
+
last_commit_id = incoming.last_commit_id
|
|
141
|
+
FROM jsonb_to_recordset(${json(this.sql, updates)}::jsonb) AS incoming(
|
|
142
|
+
object_type_id text, primary_id text, profile text, configuration text,
|
|
143
|
+
source text[], source_fingerprint text, embedding real[], last_commit_id text,
|
|
144
|
+
expected_commit_id text
|
|
145
|
+
)
|
|
146
|
+
WHERE stored.project_id = ${input.projectId}
|
|
147
|
+
AND stored.object_type_id = incoming.object_type_id AND stored.primary_id = incoming.primary_id
|
|
148
|
+
AND stored.profile = incoming.profile AND stored.last_commit_id = incoming.expected_commit_id
|
|
149
|
+
RETURNING stored.profile`
|
|
150
|
+
if (rows.length !== updates.length) throw vectorConflict()
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
async remove(input: Parameters<OntologyVectorStorage["remove"]>[0]): Promise<void> {
|
|
155
|
+
this.assertSession(input.session, input.projectId)
|
|
156
|
+
const rows = await this.sql`
|
|
157
|
+
DELETE FROM object_vectors
|
|
158
|
+
WHERE project_id = ${input.projectId}
|
|
159
|
+
AND object_type_id = ${input.ref.objectTypeId} AND primary_id = ${input.ref.primaryId}
|
|
160
|
+
AND profile = ${input.profile} AND last_commit_id = ${input.expectedCommitId}
|
|
161
|
+
RETURNING profile
|
|
162
|
+
`
|
|
163
|
+
if (rows.length !== 1) throw vectorConflict()
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function vectorConflict(): MaterializationConflictError {
|
|
168
|
+
return new MaterializationConflictError(
|
|
169
|
+
"effective-state",
|
|
170
|
+
"Vector changed since it was prepared."
|
|
171
|
+
)
|
|
172
|
+
}
|
|
@@ -188,6 +188,8 @@ export class PgAiCostStorage implements AiCostStorage {
|
|
|
188
188
|
COUNT(*)::text AS model_call_count,
|
|
189
189
|
COUNT(*) FILTER (WHERE reporting_status <> 'unavailable')::text
|
|
190
190
|
AS reported_usage_call_count,
|
|
191
|
+
COUNT(*) FILTER (WHERE input_tokens IS NOT NULL AND output_tokens IS NOT NULL)::text
|
|
192
|
+
AS complete_usage_call_count,
|
|
191
193
|
COUNT(input_tokens)::text AS input_tokens_count,
|
|
192
194
|
SUM(input_tokens)::text AS input_tokens_sum,
|
|
193
195
|
COUNT(output_tokens)::text AS output_tokens_count,
|
|
@@ -379,6 +381,7 @@ interface AggregateRow {
|
|
|
379
381
|
readonly amount_currency: string | null
|
|
380
382
|
readonly model_call_count: string
|
|
381
383
|
readonly reported_usage_call_count: string
|
|
384
|
+
readonly complete_usage_call_count: string
|
|
382
385
|
readonly input_tokens_count: string
|
|
383
386
|
readonly input_tokens_sum: string | null
|
|
384
387
|
readonly output_tokens_count: string
|
|
@@ -455,6 +458,7 @@ function aggregateFragmentFromRow(row: AggregateRow): AiAccountingAggregateFragm
|
|
|
455
458
|
modelCallCount: row.model_call_count,
|
|
456
459
|
usage: {
|
|
457
460
|
reportedCallCount: row.reported_usage_call_count,
|
|
461
|
+
completeCallCount: row.complete_usage_call_count,
|
|
458
462
|
inputTokens: meterFragment(row.input_tokens_count, row.input_tokens_sum),
|
|
459
463
|
outputTokens: meterFragment(row.output_tokens_count, row.output_tokens_sum),
|
|
460
464
|
uncachedInputTokens: meterFragment(
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DurableFileUploadSessions,
|
|
3
|
+
type FileUploadSessionPersistence,
|
|
4
|
+
type FileUploadSessionPersistenceBackend,
|
|
5
|
+
fileUploadSessionReapAt,
|
|
6
|
+
parseFileUploadSessionRow,
|
|
7
|
+
} from "@sixb/core/internal/file-upload-session-storage-provider"
|
|
8
|
+
import type { FileUploadSession, ListAbandonedFileUploadSessionsInput } from "@sixb/core/storage"
|
|
9
|
+
import { type PgStoreClient, runPgTransaction } from "./transactions"
|
|
10
|
+
|
|
11
|
+
export class PgFileUploadSessionStorage extends DurableFileUploadSessions {
|
|
12
|
+
constructor(sql: PgStoreClient) {
|
|
13
|
+
super(new PgFileUploadSessionBackend(sql))
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
class PgFileUploadSessionBackend implements FileUploadSessionPersistenceBackend {
|
|
18
|
+
constructor(private readonly sql: PgStoreClient) {}
|
|
19
|
+
|
|
20
|
+
read<T>(run: (persistence: FileUploadSessionPersistence) => Promise<T>): Promise<T> {
|
|
21
|
+
return run(new PgFileUploadSessionPersistence(this.sql, false))
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
transaction<T>(run: (persistence: FileUploadSessionPersistence) => Promise<T>): Promise<T> {
|
|
25
|
+
return runPgTransaction(this.sql, (tx) => run(new PgFileUploadSessionPersistence(tx, true)))
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
class PgFileUploadSessionPersistence implements FileUploadSessionPersistence {
|
|
30
|
+
constructor(
|
|
31
|
+
private readonly sql: PgStoreClient,
|
|
32
|
+
private readonly locking: boolean
|
|
33
|
+
) {}
|
|
34
|
+
|
|
35
|
+
async now(): Promise<Date> {
|
|
36
|
+
const [row] = await this.sql<{ readonly now: Date }[]>`SELECT clock_timestamp() AS now`
|
|
37
|
+
if (!row) throw new Error("[SixbPg] Database clock returned no row.")
|
|
38
|
+
return new Date(row.now)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async insert(session: FileUploadSession): Promise<boolean> {
|
|
42
|
+
// A unique violation would abort the surrounding transaction; report the conflict instead.
|
|
43
|
+
const rows = await this.sql`
|
|
44
|
+
INSERT INTO file_upload_sessions (
|
|
45
|
+
id, project_id, owner_principal_type, owner_principal_id, strategy, status, file_name,
|
|
46
|
+
media_type, logical_path, expected_size_bytes, expected_digest, provider_upload,
|
|
47
|
+
signed_parts, file_ref, created_at, expires_at, completed_at, aborted_at, reap_at
|
|
48
|
+
) VALUES (
|
|
49
|
+
${session.id}, ${session.projectId}, ${session.principal.type}, ${session.principal.id},
|
|
50
|
+
${session.strategy}, ${session.status}, ${session.fileName ?? null}, ${session.mediaType ?? null},
|
|
51
|
+
${session.logicalPath ?? null}, ${session.expectedSizeBytes ?? null},
|
|
52
|
+
${session.expectedDigest ?? null}, ${jsonText(session.providerUpload)}::text::jsonb,
|
|
53
|
+
${JSON.stringify(session.signedParts)}::text::jsonb, ${jsonText(session.fileRef)}::text::jsonb,
|
|
54
|
+
${session.createdAt}, ${session.expiresAt}, ${session.completedAt ?? null},
|
|
55
|
+
${session.abortedAt ?? null}, ${fileUploadSessionReapAt(session)}
|
|
56
|
+
)
|
|
57
|
+
ON CONFLICT (id) DO NOTHING
|
|
58
|
+
RETURNING id
|
|
59
|
+
`
|
|
60
|
+
return rows.length === 1
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async get(uploadId: string): Promise<FileUploadSession | null> {
|
|
64
|
+
const [row] = this.locking
|
|
65
|
+
? await this.sql<PgFileUploadSessionRow[]>`
|
|
66
|
+
SELECT * FROM file_upload_sessions WHERE id = ${uploadId} FOR UPDATE
|
|
67
|
+
`
|
|
68
|
+
: await this.sql<PgFileUploadSessionRow[]>`
|
|
69
|
+
SELECT * FROM file_upload_sessions WHERE id = ${uploadId}
|
|
70
|
+
`
|
|
71
|
+
return row ? sessionFromRow(row) : null
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
async update(session: FileUploadSession): Promise<void> {
|
|
75
|
+
await this.sql`
|
|
76
|
+
UPDATE file_upload_sessions
|
|
77
|
+
SET status = ${session.status},
|
|
78
|
+
provider_upload = ${jsonText(session.providerUpload)}::text::jsonb,
|
|
79
|
+
signed_parts = ${JSON.stringify(session.signedParts)}::text::jsonb,
|
|
80
|
+
file_ref = ${jsonText(session.fileRef)}::text::jsonb,
|
|
81
|
+
created_at = ${session.createdAt},
|
|
82
|
+
expires_at = ${session.expiresAt},
|
|
83
|
+
completed_at = ${session.completedAt ?? null},
|
|
84
|
+
aborted_at = ${session.abortedAt ?? null},
|
|
85
|
+
reap_at = ${fileUploadSessionReapAt(session)}
|
|
86
|
+
WHERE id = ${session.id}
|
|
87
|
+
`
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async listAbandoned(
|
|
91
|
+
input: ListAbandonedFileUploadSessionsInput
|
|
92
|
+
): Promise<readonly FileUploadSession[]> {
|
|
93
|
+
const rows = await this.sql<PgFileUploadSessionRow[]>`
|
|
94
|
+
SELECT * FROM file_upload_sessions
|
|
95
|
+
WHERE project_id = ${input.projectId} AND status = 'pending'
|
|
96
|
+
AND provider_upload IS NOT NULL AND expires_at <= ${input.now}
|
|
97
|
+
ORDER BY expires_at, id COLLATE "C"
|
|
98
|
+
LIMIT ${input.limit}
|
|
99
|
+
`
|
|
100
|
+
return rows.map(sessionFromRow)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async deleteReapable(now: Date): Promise<number> {
|
|
104
|
+
const rows = await this.sql`
|
|
105
|
+
DELETE FROM file_upload_sessions
|
|
106
|
+
WHERE reap_at IS NOT NULL AND reap_at <= ${now}
|
|
107
|
+
RETURNING id
|
|
108
|
+
`
|
|
109
|
+
return rows.length
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
interface PgFileUploadSessionRow {
|
|
114
|
+
readonly [column: string]: unknown
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function sessionFromRow(row: PgFileUploadSessionRow): FileUploadSession {
|
|
118
|
+
return parseFileUploadSessionRow({
|
|
119
|
+
id: row.id,
|
|
120
|
+
projectId: row.project_id,
|
|
121
|
+
ownerPrincipalType: row.owner_principal_type,
|
|
122
|
+
ownerPrincipalId: row.owner_principal_id,
|
|
123
|
+
strategy: row.strategy,
|
|
124
|
+
status: row.status,
|
|
125
|
+
fileName: row.file_name,
|
|
126
|
+
mediaType: row.media_type,
|
|
127
|
+
logicalPath: row.logical_path,
|
|
128
|
+
expectedSizeBytes: row.expected_size_bytes,
|
|
129
|
+
expectedDigest: row.expected_digest,
|
|
130
|
+
providerUpload: row.provider_upload,
|
|
131
|
+
signedParts: row.signed_parts,
|
|
132
|
+
fileRef: row.file_ref,
|
|
133
|
+
createdAt: row.created_at,
|
|
134
|
+
expiresAt: row.expires_at,
|
|
135
|
+
completedAt: row.completed_at,
|
|
136
|
+
abortedAt: row.aborted_at,
|
|
137
|
+
})
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** JSON text for a nullable jsonb column; always bind with the `::text::jsonb` double cast. */
|
|
141
|
+
function jsonText(value: object | undefined): string | null {
|
|
142
|
+
return value === undefined ? null : JSON.stringify(value)
|
|
143
|
+
}
|
|
@@ -5,8 +5,8 @@ import type {
|
|
|
5
5
|
ExpireWorkflowInterventionInput,
|
|
6
6
|
ListWorkflowInterventionsInput,
|
|
7
7
|
ListWorkflowInterventionsResult,
|
|
8
|
+
Principal,
|
|
8
9
|
SubmitWorkflowInterventionInput,
|
|
9
|
-
WorkflowInterventionActor,
|
|
10
10
|
WorkflowInterventionRecord,
|
|
11
11
|
WorkflowInterventionStorage,
|
|
12
12
|
} from "@sixb/core/storage"
|
|
@@ -78,7 +78,7 @@ export class PgWorkflowInterventionStorage implements WorkflowInterventionStorag
|
|
|
78
78
|
SET
|
|
79
79
|
status = ${"submitted"},
|
|
80
80
|
submitted_at = ${input.submittedAt ?? new Date()},
|
|
81
|
-
submitted_by = ${
|
|
81
|
+
submitted_by = ${serializePrincipal(input.submittedBy)}::text::jsonb,
|
|
82
82
|
response = ${serializeRecord(input.response)}::text::jsonb
|
|
83
83
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
84
84
|
RETURNING *
|
|
@@ -97,7 +97,7 @@ export class PgWorkflowInterventionStorage implements WorkflowInterventionStorag
|
|
|
97
97
|
SET
|
|
98
98
|
status = ${"cancelled"},
|
|
99
99
|
cancelled_at = ${input.cancelledAt ?? new Date()},
|
|
100
|
-
cancelled_by = ${
|
|
100
|
+
cancelled_by = ${serializePrincipal(input.cancelledBy)}::text::jsonb
|
|
101
101
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
102
102
|
RETURNING *
|
|
103
103
|
`
|
|
@@ -268,16 +268,12 @@ function parseRecord(value: WorkflowIOSnapshot | string): WorkflowIOSnapshot {
|
|
|
268
268
|
return typeof value === "string" ? (JSON.parse(value) as WorkflowIOSnapshot) : value
|
|
269
269
|
}
|
|
270
270
|
|
|
271
|
-
function
|
|
272
|
-
return
|
|
271
|
+
function serializePrincipal(principal: Principal | undefined): string | null {
|
|
272
|
+
return principal ? JSON.stringify(principal) : null
|
|
273
273
|
}
|
|
274
274
|
|
|
275
|
-
function
|
|
276
|
-
value
|
|
277
|
-
): WorkflowInterventionActor | undefined {
|
|
278
|
-
return typeof value === "string"
|
|
279
|
-
? (JSON.parse(value) as WorkflowInterventionActor)
|
|
280
|
-
: (value ?? undefined)
|
|
275
|
+
function parsePrincipal(value: Principal | string | null): Principal | undefined {
|
|
276
|
+
return typeof value === "string" ? (JSON.parse(value) as Principal) : (value ?? undefined)
|
|
281
277
|
}
|
|
282
278
|
|
|
283
279
|
function rowToWorkflowInterventionRecord(
|
|
@@ -299,10 +295,10 @@ function rowToWorkflowInterventionRecord(
|
|
|
299
295
|
requestedAt: new Date(row.requested_at),
|
|
300
296
|
expiresAt: row.expires_at ? new Date(row.expires_at) : undefined,
|
|
301
297
|
submittedAt: row.submitted_at ? new Date(row.submitted_at) : undefined,
|
|
302
|
-
submittedBy:
|
|
298
|
+
submittedBy: parsePrincipal(row.submitted_by),
|
|
303
299
|
response: row.response ? parseRecord(row.response) : undefined,
|
|
304
300
|
cancelledAt: row.cancelled_at ? new Date(row.cancelled_at) : undefined,
|
|
305
|
-
cancelledBy:
|
|
301
|
+
cancelledBy: parsePrincipal(row.cancelled_by),
|
|
306
302
|
expiredAt: row.expired_at ? new Date(row.expired_at) : undefined,
|
|
307
303
|
}
|
|
308
304
|
}
|
|
@@ -331,9 +327,9 @@ interface WorkflowInterventionDatabaseRow {
|
|
|
331
327
|
requested_at: Date | string
|
|
332
328
|
expires_at: Date | string | null
|
|
333
329
|
submitted_at: Date | string | null
|
|
334
|
-
submitted_by:
|
|
330
|
+
submitted_by: Principal | string | null
|
|
335
331
|
response: WorkflowIOSnapshot | string | null
|
|
336
332
|
cancelled_at: Date | string | null
|
|
337
|
-
cancelled_by:
|
|
333
|
+
cancelled_by: Principal | string | null
|
|
338
334
|
expired_at: Date | string | null
|
|
339
335
|
}
|