@sixb/pg 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -6
- package/dist/agents/agent-storage.d.ts +7 -4
- package/dist/agents/agent-storage.d.ts.map +1 -1
- package/dist/agents/checkpoints.d.ts +12 -0
- package/dist/agents/checkpoints.d.ts.map +1 -0
- package/dist/agents/index.d.ts +1 -0
- package/dist/agents/index.d.ts.map +1 -1
- package/dist/agents/messages.d.ts +2 -1
- package/dist/agents/messages.d.ts.map +1 -1
- package/dist/agents/rows.d.ts +22 -12
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/runs.d.ts +3 -2
- package/dist/agents/runs.d.ts.map +1 -1
- package/dist/auth-storage/service-accounts.d.ts.map +1 -1
- package/dist/auth-storage/shared.d.ts +0 -4
- package/dist/auth-storage/shared.d.ts.map +1 -1
- package/dist/connector-connection-storage.d.ts +9 -0
- package/dist/connector-connection-storage.d.ts.map +1 -0
- package/dist/index.d.ts +9 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5010 -1837
- package/dist/index.js.map +31 -26
- package/dist/latest-run-query.d.ts.map +1 -1
- package/dist/migrations.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts +0 -2
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/outbox.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +3 -6
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/pg-action-run-storage.d.ts +3 -2
- package/dist/pg-action-run-storage.d.ts.map +1 -1
- package/dist/pg-ai-cost-storage.d.ts +12 -0
- package/dist/pg-ai-cost-storage.d.ts.map +1 -0
- package/dist/pg-ai-usage-storage.d.ts +16 -0
- package/dist/pg-ai-usage-storage.d.ts.map +1 -0
- package/dist/pg-execution-storage.d.ts +14 -0
- package/dist/pg-execution-storage.d.ts.map +1 -0
- package/dist/pg-pipeline-run-storage.d.ts +5 -2
- package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts +5 -2
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/pg-sync-run-storage.d.ts +5 -2
- package/dist/pg-sync-run-storage.d.ts.map +1 -1
- package/dist/pg-webhook-run-storage.d.ts +5 -2
- package/dist/pg-webhook-run-storage.d.ts.map +1 -1
- package/dist/pg-workflow-run-storage.d.ts +5 -3
- package/dist/pg-workflow-run-storage.d.ts.map +1 -1
- package/dist/run-list-query.d.ts +1 -1
- package/dist/run-list-query.d.ts.map +1 -1
- package/dist/testing.d.ts +6 -0
- package/dist/testing.d.ts.map +1 -0
- package/package.json +3 -3
- package/src/agents/agent-storage.ts +9 -5
- package/src/agents/checkpoints.ts +175 -0
- package/src/agents/index.ts +1 -0
- package/src/agents/messages.ts +71 -13
- package/src/agents/rows.ts +57 -43
- package/src/agents/runs.ts +29 -15
- package/src/auth-storage/service-accounts.ts +14 -15
- package/src/auth-storage/shared.ts +0 -8
- package/src/connector-connection-storage.ts +760 -0
- package/src/index.ts +38 -15
- package/src/latest-run-query.ts +7 -1
- package/src/migrations/004-executions.sql +140 -0
- package/src/migrations/005-workflow-executions.sql +10 -0
- package/src/migrations/006-narrow-ontology-source-root-index.sql +4 -0
- package/src/migrations/007-split-overrides.sql +153 -0
- package/src/migrations/008-action-executions.sql +7 -0
- package/src/migrations/009-agent-executions.sql +21 -0
- package/src/migrations/010-ai-usage-accounting-foundation.sql +76 -0
- package/src/migrations/011-sync-failure-record.sql +23 -0
- package/src/migrations/012-pipeline-failure-record.sql +47 -0
- package/src/migrations/013-workflow-failure-record.sql +45 -0
- package/src/migrations/014-agent-failure-record.sql +56 -0
- package/src/migrations/015-projection-failure-record.sql +20 -0
- package/src/migrations/016-webhook-run-failure-record.sql +19 -0
- package/src/migrations/017-action-failure-record.sql +65 -0
- package/src/migrations/018-ontology-outbox-failure-record.sql +29 -0
- package/src/migrations/019-webhook-delivery-failure-record.sql +27 -0
- package/src/migrations/020-drop-run-usage-projections.sql +8 -0
- package/src/migrations/021-sync-pipeline-executions.sql +50 -0
- package/src/migrations/022-projection-executions.sql +115 -0
- package/src/migrations/023-webhook-executions.sql +30 -0
- package/src/migrations/024-ontology-commit-executions.sql +12 -0
- package/src/migrations/025-connector-connections.sql +231 -0
- package/src/migrations/026-ai-cost-accounting.sql +53 -0
- package/src/migrations/027-agent-context-checkpoints.sql +34 -0
- package/src/migrations/028-object-override-edit-times.sql +3 -0
- package/src/migrations.ts +82 -0
- package/src/ontology-storage/materialization-session.ts +25 -28
- package/src/ontology-storage/materialization-state.ts +161 -43
- package/src/ontology-storage/materialization-writer.ts +149 -76
- package/src/ontology-storage/materializations.ts +3 -1
- package/src/ontology-storage/outbox.ts +9 -2
- package/src/ontology-storage/shared.ts +10 -20
- package/src/pg-action-run-storage.ts +43 -67
- package/src/pg-ai-cost-storage.ts +707 -0
- package/src/pg-ai-usage-storage.ts +290 -0
- package/src/pg-execution-storage.ts +147 -0
- package/src/pg-pipeline-run-storage.ts +107 -41
- package/src/pg-projection-run-storage.ts +159 -91
- package/src/pg-sync-run-storage.ts +100 -28
- package/src/pg-webhook-run-storage.ts +89 -57
- package/src/pg-workflow-run-storage.ts +97 -130
- package/src/run-list-query.ts +9 -4
- package/src/testing.ts +18 -0
- package/dist/pg-webhook-delivery-storage.d.ts +0 -17
- package/dist/pg-webhook-delivery-storage.d.ts.map +0 -1
- package/src/pg-webhook-delivery-storage.ts +0 -180
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
CREATE TABLE connector_connection_runs (
|
|
2
|
+
project_id TEXT NOT NULL,
|
|
3
|
+
connector_id TEXT NOT NULL,
|
|
4
|
+
id TEXT NOT NULL,
|
|
5
|
+
kind TEXT NOT NULL CHECK (kind IN ('connect', 'reauthorize')),
|
|
6
|
+
slot TEXT NOT NULL,
|
|
7
|
+
initiated_by_execution_id TEXT NOT NULL,
|
|
8
|
+
status TEXT NOT NULL CHECK (status IN ('waiting', 'running', 'succeeded', 'failed', 'cancelled', 'expired')),
|
|
9
|
+
waiting_for TEXT CHECK (waiting_for IN ('provider_authorization', 'account_selection')),
|
|
10
|
+
processing_id TEXT,
|
|
11
|
+
callback_started_at TIMESTAMPTZ,
|
|
12
|
+
expires_at TIMESTAMPTZ,
|
|
13
|
+
authorization_id TEXT,
|
|
14
|
+
cleanup_authorization_id TEXT,
|
|
15
|
+
connections JSONB CHECK (connections IS NULL OR jsonb_typeof(connections) = 'array'),
|
|
16
|
+
error JSONB CHECK (error IS NULL OR jsonb_typeof(error) = 'object'),
|
|
17
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
18
|
+
updated_at TIMESTAMPTZ NOT NULL,
|
|
19
|
+
finished_at TIMESTAMPTZ,
|
|
20
|
+
PRIMARY KEY (project_id, connector_id, id),
|
|
21
|
+
UNIQUE (project_id, id),
|
|
22
|
+
CHECK (
|
|
23
|
+
(
|
|
24
|
+
status = 'waiting'
|
|
25
|
+
AND waiting_for = 'provider_authorization'
|
|
26
|
+
AND processing_id IS NULL
|
|
27
|
+
AND callback_started_at IS NULL
|
|
28
|
+
AND expires_at IS NOT NULL
|
|
29
|
+
AND authorization_id IS NULL
|
|
30
|
+
AND cleanup_authorization_id IS NULL
|
|
31
|
+
AND connections IS NULL
|
|
32
|
+
AND error IS NULL
|
|
33
|
+
AND finished_at IS NULL
|
|
34
|
+
)
|
|
35
|
+
OR (
|
|
36
|
+
status = 'waiting'
|
|
37
|
+
AND waiting_for = 'account_selection'
|
|
38
|
+
AND processing_id IS NULL
|
|
39
|
+
AND callback_started_at IS NULL
|
|
40
|
+
AND expires_at IS NOT NULL
|
|
41
|
+
AND authorization_id IS NOT NULL
|
|
42
|
+
AND cleanup_authorization_id IS NULL
|
|
43
|
+
AND connections IS NULL
|
|
44
|
+
AND error IS NULL
|
|
45
|
+
AND finished_at IS NULL
|
|
46
|
+
)
|
|
47
|
+
OR (
|
|
48
|
+
status = 'running'
|
|
49
|
+
AND waiting_for IS NULL
|
|
50
|
+
AND processing_id IS NOT NULL
|
|
51
|
+
AND callback_started_at IS NOT NULL
|
|
52
|
+
AND expires_at IS NOT NULL
|
|
53
|
+
AND cleanup_authorization_id IS NULL
|
|
54
|
+
AND connections IS NULL
|
|
55
|
+
AND error IS NULL
|
|
56
|
+
AND finished_at IS NULL
|
|
57
|
+
)
|
|
58
|
+
OR (
|
|
59
|
+
status = 'succeeded'
|
|
60
|
+
AND waiting_for IS NULL
|
|
61
|
+
AND processing_id IS NULL
|
|
62
|
+
AND callback_started_at IS NULL
|
|
63
|
+
AND expires_at IS NULL
|
|
64
|
+
AND authorization_id IS NOT NULL
|
|
65
|
+
AND connections IS NOT NULL
|
|
66
|
+
AND error IS NULL
|
|
67
|
+
AND finished_at IS NOT NULL
|
|
68
|
+
)
|
|
69
|
+
OR (
|
|
70
|
+
status = 'failed'
|
|
71
|
+
AND waiting_for IS NULL
|
|
72
|
+
AND processing_id IS NULL
|
|
73
|
+
AND callback_started_at IS NULL
|
|
74
|
+
AND expires_at IS NULL
|
|
75
|
+
AND cleanup_authorization_id IS NULL
|
|
76
|
+
AND connections IS NULL
|
|
77
|
+
AND error IS NOT NULL
|
|
78
|
+
AND finished_at IS NOT NULL
|
|
79
|
+
)
|
|
80
|
+
OR (
|
|
81
|
+
status IN ('cancelled', 'expired')
|
|
82
|
+
AND waiting_for IS NULL
|
|
83
|
+
AND processing_id IS NULL
|
|
84
|
+
AND callback_started_at IS NULL
|
|
85
|
+
AND expires_at IS NULL
|
|
86
|
+
AND cleanup_authorization_id IS NULL
|
|
87
|
+
AND connections IS NULL
|
|
88
|
+
AND error IS NULL
|
|
89
|
+
AND finished_at IS NOT NULL
|
|
90
|
+
)
|
|
91
|
+
)
|
|
92
|
+
);
|
|
93
|
+
|
|
94
|
+
CREATE INDEX idx_connector_runs_status
|
|
95
|
+
ON connector_connection_runs (project_id, connector_id, status, updated_at);
|
|
96
|
+
|
|
97
|
+
CREATE TABLE connector_authorization_attempts (
|
|
98
|
+
project_id TEXT NOT NULL,
|
|
99
|
+
connector_id TEXT NOT NULL,
|
|
100
|
+
id TEXT NOT NULL,
|
|
101
|
+
slot TEXT NOT NULL,
|
|
102
|
+
initiated_by_execution_id TEXT NOT NULL,
|
|
103
|
+
state_hash TEXT NOT NULL,
|
|
104
|
+
code_verifier JSONB NOT NULL CHECK (jsonb_typeof(code_verifier) = 'object'),
|
|
105
|
+
redirect_uri TEXT NOT NULL,
|
|
106
|
+
connection_run_id TEXT,
|
|
107
|
+
return_to TEXT,
|
|
108
|
+
callback_binding_hash TEXT,
|
|
109
|
+
reauthorization_id TEXT,
|
|
110
|
+
reauthorization_revision INTEGER CHECK (reauthorization_revision IS NULL OR reauthorization_revision >= 0),
|
|
111
|
+
reauthorization_connection_ids JSONB CHECK (
|
|
112
|
+
reauthorization_connection_ids IS NULL OR jsonb_typeof(reauthorization_connection_ids) = 'array'
|
|
113
|
+
),
|
|
114
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
115
|
+
expires_at TIMESTAMPTZ NOT NULL,
|
|
116
|
+
PRIMARY KEY (project_id, connector_id, id),
|
|
117
|
+
UNIQUE (project_id, id),
|
|
118
|
+
UNIQUE (project_id, connector_id, connection_run_id),
|
|
119
|
+
FOREIGN KEY (project_id, connector_id, connection_run_id)
|
|
120
|
+
REFERENCES connector_connection_runs (project_id, connector_id, id)
|
|
121
|
+
ON DELETE RESTRICT,
|
|
122
|
+
CHECK (
|
|
123
|
+
(connection_run_id IS NULL AND return_to IS NULL AND callback_binding_hash IS NULL)
|
|
124
|
+
OR (connection_run_id IS NOT NULL AND return_to IS NOT NULL AND callback_binding_hash IS NOT NULL)
|
|
125
|
+
),
|
|
126
|
+
CHECK (
|
|
127
|
+
(reauthorization_id IS NULL AND reauthorization_revision IS NULL AND reauthorization_connection_ids IS NULL)
|
|
128
|
+
OR (reauthorization_id IS NOT NULL AND reauthorization_revision IS NOT NULL AND reauthorization_connection_ids IS NOT NULL)
|
|
129
|
+
)
|
|
130
|
+
);
|
|
131
|
+
|
|
132
|
+
CREATE INDEX idx_connector_attempts_expiry
|
|
133
|
+
ON connector_authorization_attempts (project_id, connector_id, expires_at);
|
|
134
|
+
|
|
135
|
+
CREATE TABLE connector_authorizations (
|
|
136
|
+
project_id TEXT NOT NULL,
|
|
137
|
+
connector_id TEXT NOT NULL,
|
|
138
|
+
id TEXT NOT NULL,
|
|
139
|
+
authorized_by JSONB NOT NULL CHECK (jsonb_typeof(authorized_by) = 'object'),
|
|
140
|
+
credentials JSONB CHECK (credentials IS NULL OR jsonb_typeof(credentials) = 'object'),
|
|
141
|
+
credential_expires_at TIMESTAMPTZ,
|
|
142
|
+
scopes JSONB NOT NULL CHECK (jsonb_typeof(scopes) = 'array'),
|
|
143
|
+
accounts JSONB NOT NULL CHECK (jsonb_typeof(accounts) = 'array'),
|
|
144
|
+
status TEXT NOT NULL CHECK (status IN ('pending_selection', 'active', 'needs_reauthorization', 'revocation_pending', 'revoked')),
|
|
145
|
+
selection_expires_at TIMESTAMPTZ,
|
|
146
|
+
revision INTEGER NOT NULL CHECK (revision >= 0),
|
|
147
|
+
mutation_id TEXT,
|
|
148
|
+
mutation_kind TEXT CHECK (mutation_kind IN ('refresh', 'reauthorization', 'revocation')),
|
|
149
|
+
mutation_phase TEXT CHECK (mutation_phase IN ('prepared', 'executing', 'result_staged')),
|
|
150
|
+
mutation_holder_id TEXT,
|
|
151
|
+
mutation_expires_at TIMESTAMPTZ,
|
|
152
|
+
mutation_deadline_at TIMESTAMPTZ,
|
|
153
|
+
mutation_expected_connection_ids JSONB CHECK (
|
|
154
|
+
mutation_expected_connection_ids IS NULL OR jsonb_typeof(mutation_expected_connection_ids) = 'array'
|
|
155
|
+
),
|
|
156
|
+
staged_credentials JSONB CHECK (
|
|
157
|
+
staged_credentials IS NULL OR jsonb_typeof(staged_credentials) = 'object'
|
|
158
|
+
),
|
|
159
|
+
staged_credential_expires_at TIMESTAMPTZ,
|
|
160
|
+
staged_scopes JSONB CHECK (staged_scopes IS NULL OR jsonb_typeof(staged_scopes) = 'array'),
|
|
161
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
162
|
+
updated_at TIMESTAMPTZ NOT NULL,
|
|
163
|
+
PRIMARY KEY (project_id, connector_id, id),
|
|
164
|
+
UNIQUE (project_id, id),
|
|
165
|
+
CHECK (
|
|
166
|
+
(mutation_id IS NULL AND mutation_kind IS NULL AND mutation_phase IS NULL AND mutation_holder_id IS NULL AND mutation_expires_at IS NULL AND mutation_deadline_at IS NULL AND mutation_expected_connection_ids IS NULL AND staged_credentials IS NULL AND staged_credential_expires_at IS NULL AND staged_scopes IS NULL)
|
|
167
|
+
OR (mutation_id IS NOT NULL AND mutation_kind IS NOT NULL AND mutation_phase IS NOT NULL AND mutation_holder_id IS NOT NULL AND mutation_expires_at IS NOT NULL AND mutation_deadline_at IS NOT NULL)
|
|
168
|
+
),
|
|
169
|
+
CHECK (
|
|
170
|
+
(
|
|
171
|
+
mutation_phase IS NULL
|
|
172
|
+
AND staged_credentials IS NULL
|
|
173
|
+
AND staged_credential_expires_at IS NULL
|
|
174
|
+
AND staged_scopes IS NULL
|
|
175
|
+
)
|
|
176
|
+
OR (
|
|
177
|
+
mutation_phase IN ('prepared', 'executing')
|
|
178
|
+
AND staged_credentials IS NULL
|
|
179
|
+
AND staged_credential_expires_at IS NULL
|
|
180
|
+
AND staged_scopes IS NULL
|
|
181
|
+
)
|
|
182
|
+
OR (
|
|
183
|
+
mutation_phase = 'result_staged'
|
|
184
|
+
AND mutation_kind = 'revocation'
|
|
185
|
+
AND staged_credentials IS NULL
|
|
186
|
+
AND staged_credential_expires_at IS NULL
|
|
187
|
+
AND staged_scopes IS NULL
|
|
188
|
+
)
|
|
189
|
+
OR (
|
|
190
|
+
mutation_phase = 'result_staged'
|
|
191
|
+
AND mutation_kind IN ('refresh', 'reauthorization')
|
|
192
|
+
AND staged_credentials IS NOT NULL
|
|
193
|
+
AND staged_scopes IS NOT NULL
|
|
194
|
+
)
|
|
195
|
+
),
|
|
196
|
+
CHECK ((status = 'pending_selection') = (selection_expires_at IS NOT NULL)),
|
|
197
|
+
CHECK ((status = 'revoked') = (credentials IS NULL))
|
|
198
|
+
);
|
|
199
|
+
|
|
200
|
+
CREATE INDEX idx_connector_authorizations_status
|
|
201
|
+
ON connector_authorizations (project_id, connector_id, status, updated_at);
|
|
202
|
+
CREATE INDEX idx_connector_authorizations_mutation_expiry
|
|
203
|
+
ON connector_authorizations (project_id, connector_id, mutation_expires_at)
|
|
204
|
+
WHERE mutation_id IS NOT NULL;
|
|
205
|
+
|
|
206
|
+
CREATE TABLE connector_connections (
|
|
207
|
+
project_id TEXT NOT NULL,
|
|
208
|
+
connector_id TEXT NOT NULL,
|
|
209
|
+
id TEXT NOT NULL,
|
|
210
|
+
authorization_id TEXT NOT NULL,
|
|
211
|
+
slot TEXT NOT NULL,
|
|
212
|
+
account JSONB NOT NULL CHECK (jsonb_typeof(account) = 'object'),
|
|
213
|
+
account_id TEXT NOT NULL,
|
|
214
|
+
status TEXT NOT NULL CHECK (status IN ('connected', 'disconnected')),
|
|
215
|
+
disconnected_at TIMESTAMPTZ,
|
|
216
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
217
|
+
updated_at TIMESTAMPTZ NOT NULL,
|
|
218
|
+
PRIMARY KEY (project_id, connector_id, id),
|
|
219
|
+
UNIQUE (project_id, id),
|
|
220
|
+
UNIQUE (project_id, connector_id, slot),
|
|
221
|
+
FOREIGN KEY (project_id, connector_id, authorization_id)
|
|
222
|
+
REFERENCES connector_authorizations (project_id, connector_id, id)
|
|
223
|
+
ON DELETE RESTRICT,
|
|
224
|
+
CHECK ((status = 'connected') = (disconnected_at IS NULL)),
|
|
225
|
+
CHECK (account ? 'id' AND account ->> 'id' = account_id)
|
|
226
|
+
);
|
|
227
|
+
|
|
228
|
+
CREATE INDEX idx_connector_connections_authorization
|
|
229
|
+
ON connector_connections (project_id, connector_id, authorization_id, status, id);
|
|
230
|
+
CREATE INDEX idx_connector_connections_status
|
|
231
|
+
ON connector_connections (project_id, connector_id, status, id);
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
CREATE TABLE ai_model_call_valuations (
|
|
2
|
+
project_id TEXT NOT NULL CHECK (length(btrim(project_id)) > 0),
|
|
3
|
+
usage_record_id TEXT NOT NULL CHECK (length(btrim(usage_record_id)) > 0),
|
|
4
|
+
status TEXT NOT NULL CHECK (status IN ('rated', 'unpriceable')),
|
|
5
|
+
provider_id TEXT CHECK (provider_id IS NULL OR length(btrim(provider_id)) > 0),
|
|
6
|
+
model_id TEXT CHECK (model_id IS NULL OR length(btrim(model_id)) > 0),
|
|
7
|
+
currency TEXT CHECK (currency IS NULL OR currency ~ '^[A-Z]{3}$'),
|
|
8
|
+
amount_nanos BIGINT CHECK (amount_nanos IS NULL OR amount_nanos >= 0),
|
|
9
|
+
reason TEXT CHECK (
|
|
10
|
+
reason IS NULL OR reason IN (
|
|
11
|
+
'missingBillingIdentity',
|
|
12
|
+
'missingCatalogEntry',
|
|
13
|
+
'missingUsageMeter',
|
|
14
|
+
'unsupportedPricingDimension',
|
|
15
|
+
'invalidUsageForFormula'
|
|
16
|
+
)
|
|
17
|
+
),
|
|
18
|
+
details JSONB NOT NULL CHECK (jsonb_typeof(details) = 'object'),
|
|
19
|
+
rated_at TIMESTAMPTZ NOT NULL,
|
|
20
|
+
PRIMARY KEY (project_id, usage_record_id),
|
|
21
|
+
FOREIGN KEY (project_id, usage_record_id)
|
|
22
|
+
REFERENCES ai_model_call_usage(project_id, id)
|
|
23
|
+
ON DELETE RESTRICT,
|
|
24
|
+
CHECK (
|
|
25
|
+
(
|
|
26
|
+
status = 'rated'
|
|
27
|
+
AND provider_id IS NOT NULL
|
|
28
|
+
AND model_id IS NOT NULL
|
|
29
|
+
AND currency IS NOT NULL
|
|
30
|
+
AND amount_nanos IS NOT NULL
|
|
31
|
+
AND reason IS NULL
|
|
32
|
+
)
|
|
33
|
+
OR
|
|
34
|
+
(
|
|
35
|
+
status = 'unpriceable'
|
|
36
|
+
AND currency IS NULL
|
|
37
|
+
AND amount_nanos IS NULL
|
|
38
|
+
AND reason IS NOT NULL
|
|
39
|
+
AND (
|
|
40
|
+
(reason = 'missingBillingIdentity' AND provider_id IS NULL AND model_id IS NULL)
|
|
41
|
+
OR
|
|
42
|
+
(reason <> 'missingBillingIdentity' AND provider_id IS NOT NULL AND model_id IS NOT NULL)
|
|
43
|
+
)
|
|
44
|
+
)
|
|
45
|
+
)
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
CREATE INDEX idx_ai_model_call_usage_provider_model_time
|
|
49
|
+
ON ai_model_call_usage (project_id, provider_id, requested_model_id, occurred_at, id);
|
|
50
|
+
CREATE INDEX idx_ai_model_call_usage_model_time
|
|
51
|
+
ON ai_model_call_usage (project_id, requested_model_id, occurred_at, id);
|
|
52
|
+
CREATE INDEX idx_ai_model_call_valuations_summary
|
|
53
|
+
ON ai_model_call_valuations (project_id, status, currency, usage_record_id);
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
CREATE TABLE agent_context_checkpoints (
|
|
2
|
+
project_id TEXT NOT NULL,
|
|
3
|
+
id TEXT NOT NULL,
|
|
4
|
+
thread_id TEXT NOT NULL,
|
|
5
|
+
created_by_run_id TEXT NOT NULL,
|
|
6
|
+
previous_checkpoint_id TEXT,
|
|
7
|
+
reason TEXT NOT NULL CHECK (reason IN ('threshold', 'overflow')),
|
|
8
|
+
summary TEXT NOT NULL CHECK (length(trim(summary)) > 0),
|
|
9
|
+
summary_format_version INTEGER NOT NULL CHECK (summary_format_version = 1),
|
|
10
|
+
summarized_through_seq INTEGER NOT NULL CHECK (summarized_through_seq >= 1),
|
|
11
|
+
observed_head_seq INTEGER NOT NULL,
|
|
12
|
+
estimated_input_tokens_before BIGINT NOT NULL CHECK (estimated_input_tokens_before >= 0),
|
|
13
|
+
estimated_input_tokens_after BIGINT NOT NULL CHECK (estimated_input_tokens_after >= 0),
|
|
14
|
+
summary_model_id TEXT NOT NULL CHECK (length(trim(summary_model_id)) > 0),
|
|
15
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
16
|
+
PRIMARY KEY (project_id, id),
|
|
17
|
+
UNIQUE (project_id, created_by_run_id),
|
|
18
|
+
FOREIGN KEY (project_id, thread_id)
|
|
19
|
+
REFERENCES agent_threads(project_id, id) ON DELETE CASCADE,
|
|
20
|
+
FOREIGN KEY (project_id, created_by_run_id)
|
|
21
|
+
REFERENCES agent_runs(project_id, id) ON DELETE RESTRICT,
|
|
22
|
+
FOREIGN KEY (project_id, previous_checkpoint_id)
|
|
23
|
+
REFERENCES agent_context_checkpoints(project_id, id) ON DELETE RESTRICT,
|
|
24
|
+
CHECK (summarized_through_seq < observed_head_seq)
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
CREATE INDEX idx_agent_context_checkpoints_thread_latest
|
|
28
|
+
ON agent_context_checkpoints (
|
|
29
|
+
project_id,
|
|
30
|
+
thread_id,
|
|
31
|
+
summarized_through_seq DESC,
|
|
32
|
+
created_at DESC,
|
|
33
|
+
id DESC
|
|
34
|
+
);
|
package/src/migrations.ts
CHANGED
|
@@ -16,6 +16,63 @@ import {
|
|
|
16
16
|
import initialSchemaSql from "./migrations/001-initial-schema.sql" with { type: "text" }
|
|
17
17
|
import workflowRunOutputSql from "./migrations/002-workflow-run-output.sql" with { type: "text" }
|
|
18
18
|
import mergeSyncRunsSql from "./migrations/003-merge-sync-runs.sql" with { type: "text" }
|
|
19
|
+
import executionsSql from "./migrations/004-executions.sql" with { type: "text" }
|
|
20
|
+
import workflowExecutionsSql from "./migrations/005-workflow-executions.sql" with { type: "text" }
|
|
21
|
+
import narrowOntologySourceRootIndexSql from "./migrations/006-narrow-ontology-source-root-index.sql" with {
|
|
22
|
+
type: "text",
|
|
23
|
+
}
|
|
24
|
+
import splitOverridesSql from "./migrations/007-split-overrides.sql" with { type: "text" }
|
|
25
|
+
import actionExecutionsSql from "./migrations/008-action-executions.sql" with { type: "text" }
|
|
26
|
+
import agentExecutionsSql from "./migrations/009-agent-executions.sql" with { type: "text" }
|
|
27
|
+
import aiUsageAccountingFoundationSql from "./migrations/010-ai-usage-accounting-foundation.sql" with {
|
|
28
|
+
type: "text",
|
|
29
|
+
}
|
|
30
|
+
import syncFailureRecordSql from "./migrations/011-sync-failure-record.sql" with { type: "text" }
|
|
31
|
+
import pipelineFailureRecordSql from "./migrations/012-pipeline-failure-record.sql" with {
|
|
32
|
+
type: "text",
|
|
33
|
+
}
|
|
34
|
+
import workflowFailureRecordSql from "./migrations/013-workflow-failure-record.sql" with {
|
|
35
|
+
type: "text",
|
|
36
|
+
}
|
|
37
|
+
import agentFailureRecordSql from "./migrations/014-agent-failure-record.sql" with { type: "text" }
|
|
38
|
+
import projectionFailureRecordSql from "./migrations/015-projection-failure-record.sql" with {
|
|
39
|
+
type: "text",
|
|
40
|
+
}
|
|
41
|
+
import webhookRunFailureRecordSql from "./migrations/016-webhook-run-failure-record.sql" with {
|
|
42
|
+
type: "text",
|
|
43
|
+
}
|
|
44
|
+
import actionFailureRecordSql from "./migrations/017-action-failure-record.sql" with {
|
|
45
|
+
type: "text",
|
|
46
|
+
}
|
|
47
|
+
import ontologyOutboxFailureRecordSql from "./migrations/018-ontology-outbox-failure-record.sql" with {
|
|
48
|
+
type: "text",
|
|
49
|
+
}
|
|
50
|
+
import webhookDeliveryFailureRecordSql from "./migrations/019-webhook-delivery-failure-record.sql" with {
|
|
51
|
+
type: "text",
|
|
52
|
+
}
|
|
53
|
+
import dropRunUsageProjectionsSql from "./migrations/020-drop-run-usage-projections.sql" with {
|
|
54
|
+
type: "text",
|
|
55
|
+
}
|
|
56
|
+
import syncPipelineExecutionsSql from "./migrations/021-sync-pipeline-executions.sql" with {
|
|
57
|
+
type: "text",
|
|
58
|
+
}
|
|
59
|
+
import projectionExecutionsSql from "./migrations/022-projection-executions.sql" with {
|
|
60
|
+
type: "text",
|
|
61
|
+
}
|
|
62
|
+
import webhookExecutionsSql from "./migrations/023-webhook-executions.sql" with { type: "text" }
|
|
63
|
+
import ontologyCommitExecutionsSql from "./migrations/024-ontology-commit-executions.sql" with {
|
|
64
|
+
type: "text",
|
|
65
|
+
}
|
|
66
|
+
import connectorConnectionsSql from "./migrations/025-connector-connections.sql" with {
|
|
67
|
+
type: "text",
|
|
68
|
+
}
|
|
69
|
+
import aiCostAccountingSql from "./migrations/026-ai-cost-accounting.sql" with { type: "text" }
|
|
70
|
+
import agentContextCheckpointsSql from "./migrations/027-agent-context-checkpoints.sql" with {
|
|
71
|
+
type: "text",
|
|
72
|
+
}
|
|
73
|
+
import objectOverrideEditTimesSql from "./migrations/028-object-override-edit-times.sql" with {
|
|
74
|
+
type: "text",
|
|
75
|
+
}
|
|
19
76
|
import type { SQL, SQLClient } from "./pg-client"
|
|
20
77
|
|
|
21
78
|
export interface PostgresMigrationContext {
|
|
@@ -272,6 +329,31 @@ export const postgresStorageMigrations = defineMigrations<PostgresMigrationConte
|
|
|
272
329
|
pgSql("001-initial-schema", initialSchemaSql),
|
|
273
330
|
pgSql("002-workflow-run-output", workflowRunOutputSql),
|
|
274
331
|
pgSql("003-merge-sync-runs", mergeSyncRunsSql),
|
|
332
|
+
pgSql("004-executions", executionsSql),
|
|
333
|
+
pgSql("005-workflow-executions", workflowExecutionsSql),
|
|
334
|
+
pgSql("006-narrow-ontology-source-root-index", narrowOntologySourceRootIndexSql),
|
|
335
|
+
pgSql("007-split-overrides", splitOverridesSql),
|
|
336
|
+
pgSql("008-action-executions", actionExecutionsSql),
|
|
337
|
+
pgSql("009-agent-executions", agentExecutionsSql),
|
|
338
|
+
pgSql("010-ai-usage-accounting-foundation", aiUsageAccountingFoundationSql),
|
|
339
|
+
pgSql("011-sync-failure-record", syncFailureRecordSql),
|
|
340
|
+
pgSql("012-pipeline-failure-record", pipelineFailureRecordSql),
|
|
341
|
+
pgSql("013-workflow-failure-record", workflowFailureRecordSql),
|
|
342
|
+
pgSql("014-agent-failure-record", agentFailureRecordSql),
|
|
343
|
+
pgSql("015-projection-failure-record", projectionFailureRecordSql),
|
|
344
|
+
pgSql("016-webhook-run-failure-record", webhookRunFailureRecordSql),
|
|
345
|
+
pgSql("017-action-failure-record", actionFailureRecordSql),
|
|
346
|
+
pgSql("018-ontology-outbox-failure-record", ontologyOutboxFailureRecordSql),
|
|
347
|
+
pgSql("019-webhook-delivery-failure-record", webhookDeliveryFailureRecordSql),
|
|
348
|
+
pgSql("020-drop-run-usage-projections", dropRunUsageProjectionsSql),
|
|
349
|
+
pgSql("021-sync-pipeline-executions", syncPipelineExecutionsSql),
|
|
350
|
+
pgSql("022-projection-executions", projectionExecutionsSql),
|
|
351
|
+
pgSql("023-webhook-executions", webhookExecutionsSql),
|
|
352
|
+
pgSql("024-ontology-commit-executions", ontologyCommitExecutionsSql),
|
|
353
|
+
pgSql("025-connector-connections", connectorConnectionsSql),
|
|
354
|
+
pgSql("026-ai-cost-accounting", aiCostAccountingSql),
|
|
355
|
+
pgSql("027-agent-context-checkpoints", agentContextCheckpointsSql),
|
|
356
|
+
pgSql("028-object-override-edit-times", objectOverrideEditTimesSql),
|
|
275
357
|
],
|
|
276
358
|
})
|
|
277
359
|
|
|
@@ -19,6 +19,7 @@ import type {
|
|
|
19
19
|
StreamMaterializationWorkInput,
|
|
20
20
|
} from "@sixb/core/storage"
|
|
21
21
|
import type { SQLClient } from "../pg-client"
|
|
22
|
+
import { isUniqueViolation } from "../storage-errors"
|
|
22
23
|
import { PG_MATERIALIZATION_WORK_TABLE, PG_REPLACEMENT_WORK_TABLE } from "./materialization-state"
|
|
23
24
|
import { jsonParameter } from "./shared"
|
|
24
25
|
|
|
@@ -119,21 +120,6 @@ export class PgMaterializationSessions {
|
|
|
119
120
|
const session = this.require(input.session)
|
|
120
121
|
const records = prepareMaterializationWork(session, input)
|
|
121
122
|
if (records.length === 0) return
|
|
122
|
-
const keys = records.map(({ record }) => record.recordKey)
|
|
123
|
-
const uniqueKeys = records.map(({ uniqueKey }) => uniqueKey)
|
|
124
|
-
|
|
125
|
-
const [duplicate] = await this.sql<{ readonly record_key: string }[]>`
|
|
126
|
-
SELECT record_key
|
|
127
|
-
FROM ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}
|
|
128
|
-
WHERE session_id = ${session.id}
|
|
129
|
-
AND (
|
|
130
|
-
record_key = ANY(${this.sql.array(keys)}::text[])
|
|
131
|
-
OR unique_key = ANY(${this.sql.array(uniqueKeys)}::text[])
|
|
132
|
-
)
|
|
133
|
-
LIMIT 1
|
|
134
|
-
`
|
|
135
|
-
if (duplicate) throw duplicateWork(duplicate.record_key)
|
|
136
|
-
|
|
137
123
|
const payload = records.map(({ record, uniqueKey, columns }) => {
|
|
138
124
|
return {
|
|
139
125
|
recordKey: record.recordKey,
|
|
@@ -147,19 +133,30 @@ export class PgMaterializationSessions {
|
|
|
147
133
|
record,
|
|
148
134
|
}
|
|
149
135
|
})
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
136
|
+
// This session is the only writer to its transaction-local temp rows. Let the two unique
|
|
137
|
+
// constraints arbitrate cross-chunk duplicates instead of issuing a duplicate probe before
|
|
138
|
+
// every insert; a violation aborts the enclosing materialization transaction and is mapped
|
|
139
|
+
// back to the provider contract below.
|
|
140
|
+
try {
|
|
141
|
+
await this.sql`
|
|
142
|
+
WITH staged AS (
|
|
143
|
+
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, payload)}::jsonb)
|
|
144
|
+
)
|
|
145
|
+
INSERT INTO ${this.sql(PG_MATERIALIZATION_WORK_TABLE)} (
|
|
146
|
+
session_id, record_key, unique_key, kind, lane,
|
|
147
|
+
major_order, minor_order, sort_one, sort_two, payload
|
|
148
|
+
)
|
|
149
|
+
SELECT ${session.id}, value->>'recordKey', value->>'uniqueKey', value->>'kind',
|
|
150
|
+
value->>'lane', (value->>'majorOrder')::integer, (value->>'minorOrder')::integer,
|
|
151
|
+
value->>'sortOne', value->>'sortTwo', value->'record'
|
|
152
|
+
FROM staged
|
|
153
|
+
`
|
|
154
|
+
} catch (error) {
|
|
155
|
+
if (isUniqueViolation(error)) {
|
|
156
|
+
throw duplicateWork()
|
|
157
|
+
}
|
|
158
|
+
throw error
|
|
159
|
+
}
|
|
163
160
|
}
|
|
164
161
|
|
|
165
162
|
async *stream(input: StreamMaterializationWorkInput): AsyncIterable<MaterializationWorkPage> {
|