@sixb/sqlite 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/dist/agents/rows.d.ts +2 -0
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/threads.d.ts +2 -1
- package/dist/agents/threads.d.ts.map +1 -1
- package/dist/index.js +717 -74
- package/dist/index.js.map +20 -16
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +9 -2
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts +3 -0
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +1 -0
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/rows.d.ts +1 -0
- package/dist/objects/rows.d.ts.map +1 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/objects/vector-extension.d.ts +4 -0
- package/dist/objects/vector-extension.d.ts.map +1 -0
- package/dist/ontology-storage/index.d.ts +4 -0
- package/dist/ontology-storage/index.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +1 -0
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/outbox.d.ts.map +1 -1
- package/dist/ontology-storage/vector-indexing.d.ts +20 -0
- package/dist/ontology-storage/vector-indexing.d.ts.map +1 -0
- package/dist/ontology-storage/vectors.d.ts +16 -0
- package/dist/ontology-storage/vectors.d.ts.map +1 -0
- package/dist/vector-encoding.d.ts +3 -0
- package/dist/vector-encoding.d.ts.map +1 -0
- package/package.json +8 -4
- package/src/agents/rows.ts +12 -0
- package/src/agents/threads.ts +41 -2
- package/src/ai-cost-storage.ts +4 -0
- package/src/connector-connection-storage.ts +10 -0
- package/src/migrations/040-outbox-publication-order.sql +8 -0
- package/src/migrations/041-agent-thread-workspaces.sql +2 -0
- package/src/migrations/042-vector-profiles.sql +21 -0
- package/src/migrations/043-vector-indexing.sql +197 -0
- package/src/migrations/044-vector-batching.sql +3 -0
- package/src/migrations/045-agent-thread-sandbox-state.sql +2 -0
- package/src/migrations.ts +18 -0
- package/src/objects/query-compiler.ts +105 -7
- package/src/objects/read-scope.ts +34 -31
- package/src/objects/reader.ts +25 -1
- package/src/objects/rows.ts +5 -1
- package/src/objects/storage.ts +12 -1
- package/src/objects/vector-extension.ts +26 -0
- package/src/ontology-storage/index.ts +16 -0
- package/src/ontology-storage/materializations.ts +13 -0
- package/src/ontology-storage/outbox.ts +33 -16
- package/src/ontology-storage/vector-indexing.ts +196 -0
- package/src/ontology-storage/vectors.ts +182 -0
- package/src/vector-encoding.ts +9 -0
package/dist/index.js
CHANGED
|
@@ -3979,6 +3979,257 @@ ALTER TABLE projection_runs ADD COLUMN source_changes_read INTEGER
|
|
|
3979
3979
|
CHECK (source_changes_read IS NULL OR source_changes_read >= 0);
|
|
3980
3980
|
`;
|
|
3981
3981
|
|
|
3982
|
+
// src/migrations/040-outbox-publication-order.sql
|
|
3983
|
+
var _040_outbox_publication_order_default = `-- Match the bounded claim order without sorting the entire unpublished backlog.
|
|
3984
|
+
-- Cover every selection predicate so new databases also benefit before ANALYZE.
|
|
3985
|
+
CREATE INDEX idx_ontology_outbox_publication_order
|
|
3986
|
+
ON ontology_outbox(
|
|
3987
|
+
project_id, published_at, created_at, commit_id, commit_ordinal,
|
|
3988
|
+
available_at, lease_expires_at
|
|
3989
|
+
)
|
|
3990
|
+
WHERE published_at IS NULL;
|
|
3991
|
+
`;
|
|
3992
|
+
|
|
3993
|
+
// src/migrations/041-agent-thread-workspaces.sql
|
|
3994
|
+
var _041_agent_thread_workspaces_default = `ALTER TABLE agent_threads ADD COLUMN sandbox_params TEXT
|
|
3995
|
+
CHECK (sandbox_params IS NULL OR (json_valid(sandbox_params) AND json_type(sandbox_params) = 'object'));
|
|
3996
|
+
`;
|
|
3997
|
+
|
|
3998
|
+
// src/migrations/042-vector-profiles.sql
|
|
3999
|
+
var _042_vector_profiles_default = `CREATE TABLE object_vectors (
|
|
4000
|
+
project_id TEXT NOT NULL,
|
|
4001
|
+
object_type_id TEXT NOT NULL,
|
|
4002
|
+
primary_id TEXT NOT NULL,
|
|
4003
|
+
profile TEXT NOT NULL CHECK (length(trim(profile)) > 0),
|
|
4004
|
+
configuration TEXT NOT NULL,
|
|
4005
|
+
source TEXT NOT NULL CHECK (
|
|
4006
|
+
json_valid(source) AND json_type(source) = 'array' AND json_array_length(source) > 0
|
|
4007
|
+
),
|
|
4008
|
+
source_fingerprint TEXT NOT NULL,
|
|
4009
|
+
embedding BLOB NOT NULL CHECK (
|
|
4010
|
+
typeof(embedding) = 'blob' AND length(embedding) BETWEEN 4 AND 64000
|
|
4011
|
+
AND length(embedding) % 4 = 0
|
|
4012
|
+
),
|
|
4013
|
+
last_commit_id TEXT NOT NULL,
|
|
4014
|
+
PRIMARY KEY (project_id, object_type_id, primary_id, profile),
|
|
4015
|
+
FOREIGN KEY (project_id, object_type_id, primary_id)
|
|
4016
|
+
REFERENCES objects (project_id, object_type_id, primary_id) ON DELETE CASCADE,
|
|
4017
|
+
FOREIGN KEY (project_id, last_commit_id)
|
|
4018
|
+
REFERENCES ontology_commits (project_id, id) DEFERRABLE INITIALLY DEFERRED
|
|
4019
|
+
);
|
|
4020
|
+
`;
|
|
4021
|
+
|
|
4022
|
+
// src/migrations/043-vector-indexing.sql
|
|
4023
|
+
var _043_vector_indexing_default = `CREATE TABLE object_vector_indexing (
|
|
4024
|
+
project_id TEXT NOT NULL,
|
|
4025
|
+
id TEXT NOT NULL,
|
|
4026
|
+
object_type_id TEXT NOT NULL,
|
|
4027
|
+
primary_id TEXT NOT NULL,
|
|
4028
|
+
profile TEXT NOT NULL,
|
|
4029
|
+
request TEXT NOT NULL,
|
|
4030
|
+
status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'ready', 'failed')),
|
|
4031
|
+
available_at TEXT NOT NULL,
|
|
4032
|
+
dispatch_at TEXT NOT NULL,
|
|
4033
|
+
values_json TEXT,
|
|
4034
|
+
error TEXT,
|
|
4035
|
+
PRIMARY KEY (project_id, id),
|
|
4036
|
+
UNIQUE (project_id, object_type_id, primary_id, profile)
|
|
4037
|
+
);
|
|
4038
|
+
CREATE INDEX object_vector_indexing_due ON object_vector_indexing (project_id, dispatch_at, id)
|
|
4039
|
+
WHERE status <> 'failed';
|
|
4040
|
+
|
|
4041
|
+
CREATE TABLE executions_v2 (
|
|
4042
|
+
project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
|
|
4043
|
+
id TEXT NOT NULL CHECK (length(trim(id)) > 0),
|
|
4044
|
+
executor_kind TEXT NOT NULL CHECK (
|
|
4045
|
+
executor_kind IN (
|
|
4046
|
+
'request', 'action', 'pipeline', 'projection', 'rule', 'sync', 'webhook', 'workflow',
|
|
4047
|
+
'agent', 'kernel'
|
|
4048
|
+
)
|
|
4049
|
+
),
|
|
4050
|
+
executor_id TEXT NOT NULL CHECK (length(trim(executor_id)) > 0),
|
|
4051
|
+
source_kind TEXT NOT NULL CHECK (
|
|
4052
|
+
source_kind IN ('http', 'webhook', 'schedule', 'event', 'datasetVersion', 'execution', 'ontologyCommit')
|
|
4053
|
+
),
|
|
4054
|
+
source_id TEXT NOT NULL CHECK (length(trim(source_id)) > 0),
|
|
4055
|
+
requested_by_user_id TEXT,
|
|
4056
|
+
requested_by_service_account_id TEXT,
|
|
4057
|
+
correlation_id TEXT NOT NULL CHECK (length(trim(correlation_id)) > 0),
|
|
4058
|
+
parent_execution_id TEXT,
|
|
4059
|
+
authority_kind TEXT NOT NULL CHECK (
|
|
4060
|
+
authority_kind IN ('principal', 'trustedPrimitive', 'delegated', 'kernel', 'disabled')
|
|
4061
|
+
),
|
|
4062
|
+
authority_user_id TEXT,
|
|
4063
|
+
authority_service_account_id TEXT,
|
|
4064
|
+
authority_session_id TEXT,
|
|
4065
|
+
authority_access_token_id TEXT,
|
|
4066
|
+
authority_primitive_kind TEXT CHECK (
|
|
4067
|
+
authority_primitive_kind IS NULL
|
|
4068
|
+
OR authority_primitive_kind IN (
|
|
4069
|
+
'action', 'pipeline', 'projection', 'rule', 'sync', 'webhook', 'workflow'
|
|
4070
|
+
)
|
|
4071
|
+
),
|
|
4072
|
+
authority_primitive_id TEXT,
|
|
4073
|
+
authority_kernel_operation TEXT CHECK (
|
|
4074
|
+
authority_kernel_operation IS NULL OR authority_kernel_operation IN ('ontology.recover', 'ontology.indexVectors')
|
|
4075
|
+
),
|
|
4076
|
+
authority_delegation_kind TEXT CHECK (
|
|
4077
|
+
authority_delegation_kind IS NULL OR authority_delegation_kind = 'share'
|
|
4078
|
+
),
|
|
4079
|
+
authority_delegation_id TEXT,
|
|
4080
|
+
authority_delegation_session_id TEXT,
|
|
4081
|
+
requester_group_ids TEXT NOT NULL DEFAULT '[]',
|
|
4082
|
+
created_at TEXT NOT NULL,
|
|
4083
|
+
PRIMARY KEY (project_id, id),
|
|
4084
|
+
FOREIGN KEY (project_id, parent_execution_id) REFERENCES executions (project_id, id),
|
|
4085
|
+
FOREIGN KEY (project_id, requested_by_user_id) REFERENCES auth_users (project_id, id),
|
|
4086
|
+
FOREIGN KEY (project_id, requested_by_service_account_id)
|
|
4087
|
+
REFERENCES auth_service_accounts (project_id, id),
|
|
4088
|
+
FOREIGN KEY (project_id, authority_user_id) REFERENCES auth_users (project_id, id),
|
|
4089
|
+
FOREIGN KEY (project_id, authority_service_account_id)
|
|
4090
|
+
REFERENCES auth_service_accounts (project_id, id),
|
|
4091
|
+
FOREIGN KEY (project_id, authority_session_id) REFERENCES auth_sessions (project_id, id),
|
|
4092
|
+
FOREIGN KEY (project_id, authority_access_token_id)
|
|
4093
|
+
REFERENCES auth_access_tokens (project_id, id),
|
|
4094
|
+
FOREIGN KEY (
|
|
4095
|
+
project_id, authority_delegation_session_id, authority_delegation_id
|
|
4096
|
+
) REFERENCES share_sessions (project_id, id, grant_id),
|
|
4097
|
+
CHECK ((requested_by_user_id IS NOT NULL) + (requested_by_service_account_id IS NOT NULL) <= 1),
|
|
4098
|
+
CHECK (
|
|
4099
|
+
(source_kind = 'execution' AND parent_execution_id = source_id)
|
|
4100
|
+
OR (source_kind <> 'execution' AND parent_execution_id IS NULL)
|
|
4101
|
+
),
|
|
4102
|
+
CHECK (
|
|
4103
|
+
(
|
|
4104
|
+
authority_kind = 'principal'
|
|
4105
|
+
AND (authority_user_id IS NOT NULL) + (authority_service_account_id IS NOT NULL) = 1
|
|
4106
|
+
AND (authority_session_id IS NOT NULL) + (authority_access_token_id IS NOT NULL) <= 1
|
|
4107
|
+
AND (authority_session_id IS NULL OR authority_user_id IS NOT NULL)
|
|
4108
|
+
AND authority_primitive_kind IS NULL
|
|
4109
|
+
AND authority_primitive_id IS NULL
|
|
4110
|
+
AND authority_kernel_operation IS NULL
|
|
4111
|
+
AND authority_delegation_kind IS NULL
|
|
4112
|
+
AND authority_delegation_id IS NULL
|
|
4113
|
+
AND authority_delegation_session_id IS NULL
|
|
4114
|
+
)
|
|
4115
|
+
OR (
|
|
4116
|
+
authority_kind = 'trustedPrimitive'
|
|
4117
|
+
AND authority_user_id IS NULL
|
|
4118
|
+
AND authority_service_account_id IS NULL
|
|
4119
|
+
AND authority_session_id IS NULL
|
|
4120
|
+
AND authority_access_token_id IS NULL
|
|
4121
|
+
AND authority_primitive_kind IS NOT NULL
|
|
4122
|
+
AND authority_primitive_id IS NOT NULL
|
|
4123
|
+
AND length(trim(authority_primitive_id)) > 0
|
|
4124
|
+
AND authority_kernel_operation IS NULL
|
|
4125
|
+
AND authority_delegation_kind IS NULL
|
|
4126
|
+
AND authority_delegation_id IS NULL
|
|
4127
|
+
AND authority_delegation_session_id IS NULL
|
|
4128
|
+
)
|
|
4129
|
+
OR (
|
|
4130
|
+
authority_kind = 'delegated'
|
|
4131
|
+
AND authority_user_id IS NULL
|
|
4132
|
+
AND authority_service_account_id IS NULL
|
|
4133
|
+
AND authority_session_id IS NULL
|
|
4134
|
+
AND authority_access_token_id IS NULL
|
|
4135
|
+
AND authority_primitive_kind IS NULL
|
|
4136
|
+
AND authority_primitive_id IS NULL
|
|
4137
|
+
AND authority_kernel_operation IS NULL
|
|
4138
|
+
AND authority_delegation_kind IS NOT NULL
|
|
4139
|
+
AND authority_delegation_kind = 'share'
|
|
4140
|
+
AND authority_delegation_id IS NOT NULL
|
|
4141
|
+
AND length(trim(authority_delegation_id)) > 0
|
|
4142
|
+
AND authority_delegation_session_id IS NOT NULL
|
|
4143
|
+
AND length(trim(authority_delegation_session_id)) > 0
|
|
4144
|
+
)
|
|
4145
|
+
OR (
|
|
4146
|
+
authority_kind = 'kernel'
|
|
4147
|
+
AND authority_user_id IS NULL
|
|
4148
|
+
AND authority_service_account_id IS NULL
|
|
4149
|
+
AND authority_session_id IS NULL
|
|
4150
|
+
AND authority_access_token_id IS NULL
|
|
4151
|
+
AND authority_primitive_kind IS NULL
|
|
4152
|
+
AND authority_primitive_id IS NULL
|
|
4153
|
+
AND authority_kernel_operation IS NOT NULL
|
|
4154
|
+
AND authority_delegation_kind IS NULL
|
|
4155
|
+
AND authority_delegation_id IS NULL
|
|
4156
|
+
AND authority_delegation_session_id IS NULL
|
|
4157
|
+
)
|
|
4158
|
+
OR (
|
|
4159
|
+
authority_kind = 'disabled'
|
|
4160
|
+
AND authority_user_id IS NULL
|
|
4161
|
+
AND authority_service_account_id IS NULL
|
|
4162
|
+
AND authority_session_id IS NULL
|
|
4163
|
+
AND authority_access_token_id IS NULL
|
|
4164
|
+
AND authority_primitive_kind IS NULL
|
|
4165
|
+
AND authority_primitive_id IS NULL
|
|
4166
|
+
AND authority_kernel_operation IS NULL
|
|
4167
|
+
AND authority_delegation_kind IS NULL
|
|
4168
|
+
AND authority_delegation_id IS NULL
|
|
4169
|
+
AND authority_delegation_session_id IS NULL
|
|
4170
|
+
)
|
|
4171
|
+
),
|
|
4172
|
+
CHECK (
|
|
4173
|
+
(
|
|
4174
|
+
executor_kind = 'request'
|
|
4175
|
+
AND source_kind = 'http'
|
|
4176
|
+
AND executor_id = source_id
|
|
4177
|
+
AND (
|
|
4178
|
+
(
|
|
4179
|
+
authority_kind = 'principal'
|
|
4180
|
+
AND requested_by_user_id IS authority_user_id
|
|
4181
|
+
AND requested_by_service_account_id IS authority_service_account_id
|
|
4182
|
+
)
|
|
4183
|
+
OR (
|
|
4184
|
+
authority_kind IN ('delegated', 'disabled')
|
|
4185
|
+
AND requested_by_user_id IS NULL
|
|
4186
|
+
AND requested_by_service_account_id IS NULL
|
|
4187
|
+
)
|
|
4188
|
+
)
|
|
4189
|
+
)
|
|
4190
|
+
OR (
|
|
4191
|
+
executor_kind IN ('action', 'pipeline', 'projection', 'rule', 'sync', 'webhook', 'workflow')
|
|
4192
|
+
AND authority_kind = 'trustedPrimitive'
|
|
4193
|
+
AND executor_kind = authority_primitive_kind
|
|
4194
|
+
)
|
|
4195
|
+
OR (
|
|
4196
|
+
executor_kind = 'agent'
|
|
4197
|
+
AND source_kind = 'execution'
|
|
4198
|
+
AND (
|
|
4199
|
+
(
|
|
4200
|
+
authority_kind = 'principal'
|
|
4201
|
+
AND authority_service_account_id IS NOT NULL
|
|
4202
|
+
AND authority_session_id IS NULL
|
|
4203
|
+
AND authority_access_token_id IS NULL
|
|
4204
|
+
)
|
|
4205
|
+
OR (
|
|
4206
|
+
authority_kind = 'principal'
|
|
4207
|
+
AND authority_user_id IS NOT NULL
|
|
4208
|
+
AND (authority_session_id IS NOT NULL) + (authority_access_token_id IS NOT NULL) = 1
|
|
4209
|
+
)
|
|
4210
|
+
OR authority_kind = 'disabled'
|
|
4211
|
+
)
|
|
4212
|
+
)
|
|
4213
|
+
OR (executor_kind = 'kernel' AND authority_kind = 'kernel')
|
|
4214
|
+
)
|
|
4215
|
+
);
|
|
4216
|
+
|
|
4217
|
+
INSERT INTO executions_v2 (project_id, id, executor_kind, executor_id, source_kind, source_id, requested_by_user_id, requested_by_service_account_id, correlation_id, parent_execution_id, authority_kind, authority_user_id, authority_service_account_id, authority_session_id, authority_access_token_id, authority_primitive_kind, authority_primitive_id, authority_kernel_operation, authority_delegation_kind, authority_delegation_id, authority_delegation_session_id, requester_group_ids, created_at) SELECT project_id, id, executor_kind, executor_id, source_kind, source_id, requested_by_user_id, requested_by_service_account_id, correlation_id, parent_execution_id, authority_kind, authority_user_id, authority_service_account_id, authority_session_id, authority_access_token_id, authority_primitive_kind, authority_primitive_id, authority_kernel_operation, authority_delegation_kind, authority_delegation_id, authority_delegation_session_id, requester_group_ids, created_at FROM executions;
|
|
4218
|
+
DROP TABLE executions;
|
|
4219
|
+
ALTER TABLE executions_v2 RENAME TO executions;
|
|
4220
|
+
`;
|
|
4221
|
+
|
|
4222
|
+
// src/migrations/044-vector-batching.sql
|
|
4223
|
+
var _044_vector_batching_default = `ALTER TABLE object_vector_indexing ADD COLUMN batch_id TEXT;
|
|
4224
|
+
CREATE INDEX object_vector_indexing_batch ON object_vector_indexing (project_id, batch_id, id)
|
|
4225
|
+
WHERE batch_id IS NOT NULL;
|
|
4226
|
+
`;
|
|
4227
|
+
|
|
4228
|
+
// src/migrations/045-agent-thread-sandbox-state.sql
|
|
4229
|
+
var _045_agent_thread_sandbox_state_default = `ALTER TABLE agent_threads ADD COLUMN sandbox_state TEXT
|
|
4230
|
+
CHECK (sandbox_state IS NULL OR (json_valid(sandbox_state) AND json_type(sandbox_state) = 'object'));
|
|
4231
|
+
`;
|
|
4232
|
+
|
|
3982
4233
|
// src/migrations/037-execution-requester-groups.sql
|
|
3983
4234
|
var _037_execution_requester_groups_default = `ALTER TABLE executions ADD COLUMN requester_group_ids TEXT NOT NULL DEFAULT '[]';
|
|
3984
4235
|
|
|
@@ -4134,7 +4385,13 @@ var sqliteStorageMigrations = defineMigrations({
|
|
|
4134
4385
|
sqliteSql("036-share-sessions", _036_share_sessions_default),
|
|
4135
4386
|
sqliteSql("037-execution-requester-groups", _037_execution_requester_groups_default),
|
|
4136
4387
|
sqliteSql("038-projection-source-roots", _038_projection_source_roots_default),
|
|
4137
|
-
sqliteSql("039-connector-optional-pkce", _039_connector_optional_pkce_default)
|
|
4388
|
+
sqliteSql("039-connector-optional-pkce", _039_connector_optional_pkce_default),
|
|
4389
|
+
sqliteSql("040-outbox-publication-order", _040_outbox_publication_order_default),
|
|
4390
|
+
sqliteSql("041-agent-thread-workspaces", _041_agent_thread_workspaces_default),
|
|
4391
|
+
sqliteSql("042-vector-profiles", _042_vector_profiles_default),
|
|
4392
|
+
sqliteSql("043-vector-indexing", _043_vector_indexing_default),
|
|
4393
|
+
sqliteSql("044-vector-batching", _044_vector_batching_default),
|
|
4394
|
+
sqliteSql("045-agent-thread-sandbox-state", _045_agent_thread_sandbox_state_default)
|
|
4138
4395
|
]
|
|
4139
4396
|
});
|
|
4140
4397
|
function sqliteStoragePath(basePath) {
|
|
@@ -4819,6 +5076,7 @@ import {
|
|
|
4819
5076
|
} from "@sixb/core/storage";
|
|
4820
5077
|
|
|
4821
5078
|
// src/agents/rows.ts
|
|
5079
|
+
import { snapshotAgentThreadSandboxParams } from "@sixb/core/internal/agent-run-storage-provider";
|
|
4822
5080
|
import { parseSixbFailure } from "@sixb/core/internal/errors";
|
|
4823
5081
|
import {
|
|
4824
5082
|
AGENT_RUN_FAILURE_CODES,
|
|
@@ -4830,6 +5088,10 @@ function rowToThreadRecord(row) {
|
|
|
4830
5088
|
projectId: row.project_id,
|
|
4831
5089
|
ownerPrincipal: { type: row.owner_principal_type, id: row.owner_principal_id },
|
|
4832
5090
|
title: row.title ?? undefined,
|
|
5091
|
+
...row.sandbox_state == null ? {} : {
|
|
5092
|
+
sandboxState: JSON.parse(row.sandbox_state)
|
|
5093
|
+
},
|
|
5094
|
+
...row.sandbox_params == null ? {} : { sandboxParams: snapshotAgentThreadSandboxParams(JSON.parse(row.sandbox_params)) },
|
|
4833
5095
|
status: row.status,
|
|
4834
5096
|
activeRunId: row.active_run_id,
|
|
4835
5097
|
lastMessageAt: row.last_message_at ? new Date(row.last_message_at) : undefined,
|
|
@@ -5529,6 +5791,10 @@ class SqliteAgentRunStore {
|
|
|
5529
5791
|
}
|
|
5530
5792
|
|
|
5531
5793
|
// src/agents/threads.ts
|
|
5794
|
+
import {
|
|
5795
|
+
snapshotAgentThreadSandboxParams as snapshotAgentThreadSandboxParams2,
|
|
5796
|
+
transitionAgentThreadSandbox
|
|
5797
|
+
} from "@sixb/core/internal/agent-run-storage-provider";
|
|
5532
5798
|
import {
|
|
5533
5799
|
AgentStorageError as AgentStorageError4
|
|
5534
5800
|
} from "@sixb/core/storage";
|
|
@@ -5537,7 +5803,17 @@ class SqliteAgentThreadStore {
|
|
|
5537
5803
|
constructor(db) {
|
|
5538
5804
|
this.db = db;
|
|
5539
5805
|
}
|
|
5806
|
+
async transitionSandbox(input) {
|
|
5807
|
+
return this.db.transaction(() => {
|
|
5808
|
+
const run = input.action === "recreate" ? null : this.db.query("SELECT * FROM agent_runs WHERE project_id = ? AND id = ?").get(input.projectId, input.runId);
|
|
5809
|
+
const thread = this.requireThread(input.projectId, input.id);
|
|
5810
|
+
const state = transitionAgentThreadSandbox(thread, run ? rowToRunRecord(run) : null, input);
|
|
5811
|
+
this.db.query("UPDATE agent_threads SET sandbox_state = ? WHERE project_id = ? AND id = ?").run(JSON.stringify(state), input.projectId, input.id);
|
|
5812
|
+
return state;
|
|
5813
|
+
}).immediate();
|
|
5814
|
+
}
|
|
5540
5815
|
async create(input) {
|
|
5816
|
+
const sandboxParams = input.sandboxParams === undefined ? null : JSON.stringify(snapshotAgentThreadSandboxParams2(input.sandboxParams));
|
|
5541
5817
|
const createdAt = input.createdAt ?? new Date;
|
|
5542
5818
|
const updatedAt = input.updatedAt ?? createdAt;
|
|
5543
5819
|
try {
|
|
@@ -5548,14 +5824,15 @@ class SqliteAgentThreadStore {
|
|
|
5548
5824
|
owner_principal_type,
|
|
5549
5825
|
owner_principal_id,
|
|
5550
5826
|
title,
|
|
5827
|
+
sandbox_params,
|
|
5551
5828
|
status,
|
|
5552
5829
|
active_run_id,
|
|
5553
5830
|
last_message_at,
|
|
5554
5831
|
message_count,
|
|
5555
5832
|
created_at,
|
|
5556
5833
|
updated_at
|
|
5557
|
-
) VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, 0, ?, ?)
|
|
5558
|
-
`).run(input.projectId, input.id, input.ownerPrincipal.type, input.ownerPrincipal.id, input.title ?? null, input.status ?? "active", createdAt.toISOString(), updatedAt.toISOString());
|
|
5834
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, NULL, 0, ?, ?)
|
|
5835
|
+
`).run(input.projectId, input.id, input.ownerPrincipal.type, input.ownerPrincipal.id, input.title ?? null, sandboxParams, input.status ?? "active", createdAt.toISOString(), updatedAt.toISOString());
|
|
5559
5836
|
} catch (error) {
|
|
5560
5837
|
if (isUniqueConstraintError2(error)) {
|
|
5561
5838
|
throw new AgentStorageError4("duplicate_id", `[SixbSqlite] Agent thread '${input.id}' already exists for project '${input.projectId}'.`);
|
|
@@ -6176,6 +6453,8 @@ var SQLITE_ACCOUNTING_OVERVIEW_SQL = `
|
|
|
6176
6453
|
CAST(COUNT(*) AS TEXT) AS model_call_count,
|
|
6177
6454
|
CAST(SUM(CASE WHEN reporting_status <> 'unavailable' THEN 1 ELSE 0 END) AS TEXT)
|
|
6178
6455
|
AS reported_usage_call_count,
|
|
6456
|
+
CAST(SUM(CASE WHEN input_tokens IS NOT NULL AND output_tokens IS NOT NULL THEN 1 ELSE 0 END) AS TEXT)
|
|
6457
|
+
AS complete_usage_call_count,
|
|
6179
6458
|
CAST(COUNT(input_tokens) AS TEXT) AS input_tokens_count,
|
|
6180
6459
|
CAST(SUM(input_tokens) AS TEXT) AS input_tokens_sum,
|
|
6181
6460
|
CAST(COUNT(output_tokens) AS TEXT) AS output_tokens_count,
|
|
@@ -6228,6 +6507,7 @@ function aggregateFragmentFromRow(row) {
|
|
|
6228
6507
|
modelCallCount: row.model_call_count,
|
|
6229
6508
|
usage: {
|
|
6230
6509
|
reportedCallCount: row.reported_usage_call_count,
|
|
6510
|
+
completeCallCount: row.complete_usage_call_count,
|
|
6231
6511
|
inputTokens: meterFragment(row.input_tokens_count, row.input_tokens_sum),
|
|
6232
6512
|
outputTokens: meterFragment(row.output_tokens_count, row.output_tokens_sum),
|
|
6233
6513
|
uncachedInputTokens: meterFragment(row.uncached_input_tokens_count, row.uncached_input_tokens_sum),
|
|
@@ -9152,6 +9432,14 @@ class SqliteConnectorConnectionPersistence {
|
|
|
9152
9432
|
throw error;
|
|
9153
9433
|
}
|
|
9154
9434
|
}
|
|
9435
|
+
async listPendingConnectionRuns() {
|
|
9436
|
+
if (!this.scope.connectorId)
|
|
9437
|
+
return [];
|
|
9438
|
+
const rows = this.db.query(`SELECT * FROM connector_connection_runs
|
|
9439
|
+
WHERE project_id = ? AND connector_id = ?
|
|
9440
|
+
AND (status = 'running' OR (status = 'waiting' AND waiting_for = 'account_selection'))`).all(this.scope.projectId, this.scope.connectorId);
|
|
9441
|
+
return rows.map(connectionRunFromRow);
|
|
9442
|
+
}
|
|
9155
9443
|
async getConnectionRun(id) {
|
|
9156
9444
|
const row = this.getById("connector_connection_runs", id);
|
|
9157
9445
|
return row ? connectionRunFromRow(row) : null;
|
|
@@ -9632,13 +9920,37 @@ function isDuplicateExecutionError(error) {
|
|
|
9632
9920
|
return error instanceof Error && error.message.includes("UNIQUE constraint failed: executions.project_id, executions.id");
|
|
9633
9921
|
}
|
|
9634
9922
|
|
|
9923
|
+
// src/objects/storage.ts
|
|
9924
|
+
import { hasVectorProfile as hasVectorProfile3 } from "@sixb/core/internal/query";
|
|
9925
|
+
|
|
9635
9926
|
// src/objects/query-compiler.ts
|
|
9636
9927
|
import {
|
|
9637
9928
|
ObjectQueryExecutionError
|
|
9638
9929
|
} from "@sixb/core";
|
|
9930
|
+
import {
|
|
9931
|
+
hasVectorProfile,
|
|
9932
|
+
isVectorProfileQuery,
|
|
9933
|
+
normalizeVector,
|
|
9934
|
+
vectorSearchCandidateLimit
|
|
9935
|
+
} from "@sixb/core/internal/query";
|
|
9936
|
+
|
|
9937
|
+
// src/vector-encoding.ts
|
|
9938
|
+
function encodeSqliteVector(values) {
|
|
9939
|
+
const bytes = new Uint8Array(values.length * 4);
|
|
9940
|
+
const view = new DataView(bytes.buffer);
|
|
9941
|
+
for (let index = 0;index < values.length; index++) {
|
|
9942
|
+
view.setFloat32(index * 4, values[index], true);
|
|
9943
|
+
}
|
|
9944
|
+
return bytes;
|
|
9945
|
+
}
|
|
9946
|
+
|
|
9947
|
+
// src/objects/query-compiler.ts
|
|
9639
9948
|
var PAGE_TOKEN_PREFIX = "keyset:";
|
|
9640
9949
|
var DEFAULT_EXPANSION_FANOUT = 1000;
|
|
9641
9950
|
function compileObjectQuery(projectId, query, options = {}) {
|
|
9951
|
+
if (hasVectorProfile(query) && !isVectorProfileQuery(query)) {
|
|
9952
|
+
throw new ObjectQueryExecutionError("unsupported_vector_composition", "Vector search supports one profile with filters before ranking and limit/project after.");
|
|
9953
|
+
}
|
|
9642
9954
|
const source = options.source ?? DEFAULT_OBJECT_QUERY_SOURCE;
|
|
9643
9955
|
const ctx = { probeLimit: options.includeTotal === false, source };
|
|
9644
9956
|
return source.wrapQuery(compileObjectQueryInternal(projectId, query, ctx));
|
|
@@ -9677,8 +9989,59 @@ function compileObjectQueryInternal(projectId, query, ctx) {
|
|
|
9677
9989
|
case "expand":
|
|
9678
9990
|
return compileExpand(projectId, query.input, query.expansions, ctx);
|
|
9679
9991
|
case "vector":
|
|
9680
|
-
|
|
9681
|
-
}
|
|
9992
|
+
return compileVector(projectId, query, ctx);
|
|
9993
|
+
}
|
|
9994
|
+
}
|
|
9995
|
+
function compileVector(projectId, query, ctx) {
|
|
9996
|
+
if (typeof query.vector === "string" || !isVectorProfileQuery(query) || !query.configuration || !Number.isSafeInteger(query.k) || query.k < 1 || query.k > 1000 || query.vector.length < 1 || query.vector.length > 16000) {
|
|
9997
|
+
throw new ObjectQueryExecutionError("invalid_vector_query", "A validated named vector profile is required.");
|
|
9998
|
+
}
|
|
9999
|
+
const input = compileObjectQueryInternal(projectId, query.input, exactContext(ctx));
|
|
10000
|
+
const permissions = ctx.source.objectPropertyPermissionsTable;
|
|
10001
|
+
const permissionJoin = permissions ? `CROSS JOIN (
|
|
10002
|
+
SELECT project_id, object_type_id, primary_id, json_group_array(property_id) AS property_ids
|
|
10003
|
+
FROM ${permissions}
|
|
10004
|
+
GROUP BY project_id, object_type_id, primary_id
|
|
10005
|
+
) AS vector_permissions ON vector_permissions.project_id = vectors.project_id
|
|
10006
|
+
AND vector_permissions.object_type_id = vectors.object_type_id
|
|
10007
|
+
AND vector_permissions.primary_id = vectors.primary_id` : "";
|
|
10008
|
+
const authorized = permissions ? `AND NOT EXISTS (
|
|
10009
|
+
SELECT 1 FROM json_each(vectors.source) AS required
|
|
10010
|
+
WHERE required.value NOT IN (SELECT value FROM json_each(vector_permissions.property_ids))
|
|
10011
|
+
)` : "";
|
|
10012
|
+
const candidates = `FROM (${input.sql}) AS input
|
|
10013
|
+
CROSS JOIN object_vectors AS vectors ON vectors.project_id = input.project_id
|
|
10014
|
+
AND vectors.object_type_id = input.object_type_id AND vectors.primary_id = input.primary_id
|
|
10015
|
+
${permissionJoin}
|
|
10016
|
+
WHERE vectors.profile = ? AND vectors.configuration = ?
|
|
10017
|
+
AND length(vectors.embedding) / 4 = ? ${authorized}`;
|
|
10018
|
+
const candidateArgs = [...input.args, query.profile, query.configuration, query.vector.length];
|
|
10019
|
+
const limit = vectorSearchCandidateLimit(query.vector.length);
|
|
10020
|
+
const values = normalizeVector(query.vector, query.vector.length);
|
|
10021
|
+
const order = compileOrder([
|
|
10022
|
+
{ kind: "column", column: "_vector_score", direction: "desc" },
|
|
10023
|
+
...identityOrderFields().map((field) => ({ ...field, binary: true }))
|
|
10024
|
+
]);
|
|
10025
|
+
const sql = `SELECT * FROM (SELECT input.*, max(-1, min(1, 1 - vec_distance_cosine(vectors.embedding, ?))) AS _vector_score
|
|
10026
|
+
${candidates}) AS ranked ORDER BY ${order.sql} LIMIT ?`;
|
|
10027
|
+
const args = [encodeSqliteVector(values), ...candidateArgs, query.k];
|
|
10028
|
+
return {
|
|
10029
|
+
sql,
|
|
10030
|
+
args,
|
|
10031
|
+
totalSql: `SELECT COUNT(*) AS total FROM (SELECT 1 ${candidates} LIMIT ?) AS eligible`,
|
|
10032
|
+
totalArgs: [...candidateArgs, query.k],
|
|
10033
|
+
order,
|
|
10034
|
+
vectorProbe: {
|
|
10035
|
+
sql: `SELECT COUNT(*) AS total FROM (SELECT 1 ${candidates} LIMIT ?) AS candidates`,
|
|
10036
|
+
args: [...candidateArgs, limit + 1],
|
|
10037
|
+
limit
|
|
10038
|
+
},
|
|
10039
|
+
hasMore: () => false,
|
|
10040
|
+
trimRows: identityRows,
|
|
10041
|
+
nextPageToken: () => {
|
|
10042
|
+
return;
|
|
10043
|
+
}
|
|
10044
|
+
};
|
|
9682
10045
|
}
|
|
9683
10046
|
function compileStart(projectId, query, ctx) {
|
|
9684
10047
|
if (query.includeSubtypes === true) {
|
|
@@ -9768,6 +10131,7 @@ function compileWhere(projectId, inputQuery, predicate, ctx) {
|
|
|
9768
10131
|
WHERE ${predicate.sql}
|
|
9769
10132
|
`,
|
|
9770
10133
|
totalArgs: [...input.args, ...predicate.args],
|
|
10134
|
+
vectorProbe: input.vectorProbe,
|
|
9771
10135
|
order: input.order,
|
|
9772
10136
|
hasMore: () => false,
|
|
9773
10137
|
trimRows: identityRows,
|
|
@@ -9798,7 +10162,7 @@ function compileSort(projectId, inputQuery, fields, ctx) {
|
|
|
9798
10162
|
input.created_at,
|
|
9799
10163
|
input.updated_at,
|
|
9800
10164
|
input.version,
|
|
9801
|
-
input.last_commit_id
|
|
10165
|
+
input.last_commit_id${input.vectorProbe ? ", input._vector_score" : ""}
|
|
9802
10166
|
FROM (${input.sql}) AS input
|
|
9803
10167
|
ORDER BY ${order.sql}
|
|
9804
10168
|
`,
|
|
@@ -9829,6 +10193,7 @@ function compileLimit(projectId, inputQuery, rawLimit, ctx) {
|
|
|
9829
10193
|
FROM (${input.sql}) AS input
|
|
9830
10194
|
`,
|
|
9831
10195
|
totalArgs: input.args,
|
|
10196
|
+
vectorProbe: input.vectorProbe,
|
|
9832
10197
|
order: input.order,
|
|
9833
10198
|
hasMore: (rowCount, total) => total === undefined ? ctx.probeLimit && rowCount > limit : limit < total,
|
|
9834
10199
|
trimRows: ctx.probeLimit ? (rows) => rows.slice(0, limit) : identityRows,
|
|
@@ -9857,6 +10222,7 @@ function compilePage(projectId, inputQuery, rawPageSize, pageToken, ctx) {
|
|
|
9857
10222
|
FROM (${input.sql}) AS input
|
|
9858
10223
|
`,
|
|
9859
10224
|
totalArgs: input.args,
|
|
10225
|
+
vectorProbe: input.vectorProbe,
|
|
9860
10226
|
order: input.order,
|
|
9861
10227
|
hasMore: (rowCount) => rowCount > pageSize,
|
|
9862
10228
|
trimRows: (rows) => rows.slice(0, pageSize),
|
|
@@ -9933,6 +10299,7 @@ function compileExpand(projectId, inputQuery, expansions, ctx) {
|
|
|
9933
10299
|
args: [...expand.args, ...input.args, ...inputOrder.args],
|
|
9934
10300
|
totalSql: input.totalSql,
|
|
9935
10301
|
totalArgs: input.totalArgs,
|
|
10302
|
+
vectorProbe: input.vectorProbe,
|
|
9936
10303
|
order: input.order,
|
|
9937
10304
|
hasMoreProbe: input.hasMoreProbe,
|
|
9938
10305
|
hasMore: input.hasMore,
|
|
@@ -10103,13 +10470,14 @@ function compileProject(projectId, inputQuery, properties, ctx) {
|
|
|
10103
10470
|
input.created_at,
|
|
10104
10471
|
input.updated_at,
|
|
10105
10472
|
input.version,
|
|
10106
|
-
input.last_commit_id
|
|
10473
|
+
input.last_commit_id${input.vectorProbe ? ", input._vector_score" : ""}
|
|
10107
10474
|
FROM (${input.sql}) AS input
|
|
10108
10475
|
ORDER BY ${inputOrder.sql}
|
|
10109
10476
|
`,
|
|
10110
10477
|
args: [...projection.args, ...input.args, ...inputOrder.args],
|
|
10111
10478
|
totalSql: input.totalSql,
|
|
10112
10479
|
totalArgs: input.totalArgs,
|
|
10480
|
+
vectorProbe: input.vectorProbe,
|
|
10113
10481
|
order: outputOrder,
|
|
10114
10482
|
hasMoreProbe: input.hasMoreProbe,
|
|
10115
10483
|
hasMore: input.hasMore,
|
|
@@ -10304,7 +10672,7 @@ function compileOrder(fields, qualifier, propertyColumn = "properties") {
|
|
|
10304
10672
|
const args = [];
|
|
10305
10673
|
for (const field of fields) {
|
|
10306
10674
|
if (field.kind === "column") {
|
|
10307
|
-
clauses.push(`${columnExpression(field.column, qualifier)} ${field.direction.toUpperCase()}`);
|
|
10675
|
+
clauses.push(`${columnExpression(field.column, qualifier)}${field.binary ? " COLLATE BINARY" : ""} ${field.direction.toUpperCase()}`);
|
|
10308
10676
|
continue;
|
|
10309
10677
|
}
|
|
10310
10678
|
const direction = field.direction === "desc" ? "DESC" : "ASC";
|
|
@@ -10490,6 +10858,8 @@ function isPlainObject(value) {
|
|
|
10490
10858
|
}
|
|
10491
10859
|
|
|
10492
10860
|
// src/objects/reader.ts
|
|
10861
|
+
import { ObjectQueryExecutionError as ObjectQueryExecutionError3 } from "@sixb/core";
|
|
10862
|
+
import { hasVectorProfile as hasVectorProfile2 } from "@sixb/core/internal/query";
|
|
10493
10863
|
import { linkBatchKey, objectBatchKey } from "@sixb/core/storage";
|
|
10494
10864
|
|
|
10495
10865
|
// src/objects/rows.ts
|
|
@@ -10506,7 +10876,10 @@ function rowToObject(row) {
|
|
|
10506
10876
|
};
|
|
10507
10877
|
}
|
|
10508
10878
|
function queryRowToObject(row) {
|
|
10509
|
-
const base =
|
|
10879
|
+
const base = {
|
|
10880
|
+
...rowToObject(row),
|
|
10881
|
+
...row._vector_score === undefined ? {} : { score: row._vector_score }
|
|
10882
|
+
};
|
|
10510
10883
|
const links = reviveExpandedLinks(parseExpandColumn(row._expand));
|
|
10511
10884
|
return links ? { ...base, links } : base;
|
|
10512
10885
|
}
|
|
@@ -10569,6 +10942,25 @@ function isPlainObject2(value) {
|
|
|
10569
10942
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
10570
10943
|
}
|
|
10571
10944
|
|
|
10945
|
+
// src/objects/vector-extension.ts
|
|
10946
|
+
import { ObjectQueryExecutionError as ObjectQueryExecutionError2 } from "@sixb/core";
|
|
10947
|
+
import { load } from "sqlite-vec";
|
|
10948
|
+
var loaded = new WeakSet;
|
|
10949
|
+
function ensureSqliteVectorSearch(db) {
|
|
10950
|
+
if (loaded.has(db))
|
|
10951
|
+
return;
|
|
10952
|
+
const options = db.query("PRAGMA compile_options").all();
|
|
10953
|
+
if (options.some((option) => option.compile_options === "OMIT_LOAD_EXTENSION")) {
|
|
10954
|
+
throw new ObjectQueryExecutionError2("vector_extension_missing", "SQLite vector search requires extension support. On macOS install SQLite (brew install sqlite), then call Database.setCustomSQLite with its library path before opening any database.");
|
|
10955
|
+
}
|
|
10956
|
+
try {
|
|
10957
|
+
load(db);
|
|
10958
|
+
loaded.add(db);
|
|
10959
|
+
} catch (cause) {
|
|
10960
|
+
throw new Error("[SixbSqlite] Could not load sqlite-vec for vector search. Check that its native package supports your platform.", { cause });
|
|
10961
|
+
}
|
|
10962
|
+
}
|
|
10963
|
+
|
|
10572
10964
|
// src/objects/reader.ts
|
|
10573
10965
|
class SqliteObjectReader {
|
|
10574
10966
|
db;
|
|
@@ -10577,7 +10969,21 @@ class SqliteObjectReader {
|
|
|
10577
10969
|
this.db = db;
|
|
10578
10970
|
this.source = source;
|
|
10579
10971
|
}
|
|
10972
|
+
assertVectorSearch(params) {
|
|
10973
|
+
if (!hasVectorProfile2(params.query))
|
|
10974
|
+
return;
|
|
10975
|
+
const compiled = compileObjectQuery(params.projectId, params.query, { source: this.source });
|
|
10976
|
+
if (!compiled.vectorProbe)
|
|
10977
|
+
return;
|
|
10978
|
+
ensureSqliteVectorSearch(this.db);
|
|
10979
|
+
const probe = compiled.vectorProbe;
|
|
10980
|
+
const row = this.db.query(probe.sql).get(...probe.args);
|
|
10981
|
+
if (Number(row?.total ?? 0) > probe.limit) {
|
|
10982
|
+
throw new ObjectQueryExecutionError3("vector_work_limit_exceeded", `Exact vector search supports at most ${probe.limit} eligible vectors at this dimension. Narrow the query filters.`);
|
|
10983
|
+
}
|
|
10984
|
+
}
|
|
10580
10985
|
queryObjects(params) {
|
|
10986
|
+
this.assertVectorSearch(params);
|
|
10581
10987
|
const source = this.source;
|
|
10582
10988
|
const compiled = compileObjectQuery(params.projectId, params.query, {
|
|
10583
10989
|
includeTotal: params.includeTotal,
|
|
@@ -10595,6 +11001,7 @@ class SqliteObjectReader {
|
|
|
10595
11001
|
};
|
|
10596
11002
|
}
|
|
10597
11003
|
countObjects(params) {
|
|
11004
|
+
this.assertVectorSearch(params);
|
|
10598
11005
|
const source = this.source;
|
|
10599
11006
|
return {
|
|
10600
11007
|
count: readTotal(this.db, compileObjectQuery(params.projectId, stripOuterRowShape(params.query), {
|
|
@@ -10603,6 +11010,7 @@ class SqliteObjectReader {
|
|
|
10603
11010
|
};
|
|
10604
11011
|
}
|
|
10605
11012
|
existsObjects(params) {
|
|
11013
|
+
this.assertVectorSearch(params);
|
|
10606
11014
|
const source = this.source;
|
|
10607
11015
|
const compiled = compileObjectQuery(params.projectId, existsProbeQuery(params.query), {
|
|
10608
11016
|
source
|
|
@@ -10610,6 +11018,7 @@ class SqliteObjectReader {
|
|
|
10610
11018
|
return { exists: this.db.query(compiled.sql).get(...compiled.args) !== null };
|
|
10611
11019
|
}
|
|
10612
11020
|
facetObjects(params) {
|
|
11021
|
+
this.assertVectorSearch(params);
|
|
10613
11022
|
const source = this.source;
|
|
10614
11023
|
const compiled = compileObjectQuery(params.projectId, stripOuterRowShape(params.query), {
|
|
10615
11024
|
source
|
|
@@ -10946,6 +11355,7 @@ ${sql}`,
|
|
|
10946
11355
|
wrapQuery: (query) => ({
|
|
10947
11356
|
...query,
|
|
10948
11357
|
...wrapStatement(query.sql, query.args),
|
|
11358
|
+
vectorProbe: query.vectorProbe ? { ...query.vectorProbe, ...wrapStatement(query.vectorProbe.sql, query.vectorProbe.args) } : undefined,
|
|
10949
11359
|
totalSql: `${compiled.sql}
|
|
10950
11360
|
${query.totalSql}`,
|
|
10951
11361
|
totalArgs: [...compiled.args, ...query.totalArgs],
|
|
@@ -11032,16 +11442,16 @@ function compileSelectedReadScopeCte(scopeJson, projectId) {
|
|
|
11032
11442
|
|
|
11033
11443
|
SELECT edge.project_id, selection.node_id, edge.target_type_id, edge.target_id
|
|
11034
11444
|
FROM _sixb_scope_reachable AS parent
|
|
11035
|
-
JOIN _sixb_scope_link_selections AS selection
|
|
11445
|
+
CROSS JOIN _sixb_scope_link_selections AS selection
|
|
11036
11446
|
ON selection.parent_node_id = parent.node_id
|
|
11037
11447
|
AND selection.source_object_type_id = parent.object_type_id
|
|
11038
|
-
JOIN links AS edge
|
|
11448
|
+
CROSS JOIN links AS edge
|
|
11039
11449
|
ON edge.project_id = parent.project_id
|
|
11040
11450
|
AND edge.source_type_id = parent.object_type_id
|
|
11041
11451
|
AND edge.source_id = parent.primary_id
|
|
11042
11452
|
AND edge.link_id = selection.link_id
|
|
11043
11453
|
AND edge.target_type_id = selection.target_object_type_id
|
|
11044
|
-
JOIN objects AS target
|
|
11454
|
+
CROSS JOIN objects AS target
|
|
11045
11455
|
ON target.project_id = edge.project_id
|
|
11046
11456
|
AND target.object_type_id = edge.target_type_id
|
|
11047
11457
|
AND target.primary_id = edge.target_id
|
|
@@ -11062,32 +11472,29 @@ function compileSelectedReadScopeCte(scopeJson, projectId) {
|
|
|
11062
11472
|
SELECT DISTINCT project_id, object_type_id, primary_id
|
|
11063
11473
|
FROM _sixb_scope_reachable
|
|
11064
11474
|
),
|
|
11475
|
+
_sixb_scope_object_grants AS (
|
|
11476
|
+
SELECT reachable.project_id, reachable.object_type_id, reachable.primary_id,
|
|
11477
|
+
json_group_array(DISTINCT selected_property.value) FILTER (WHERE selected_property.value IS NOT NULL) AS property_ids
|
|
11478
|
+
FROM _sixb_scope_reachable AS reachable
|
|
11479
|
+
LEFT JOIN _sixb_scope_object_selections AS selection
|
|
11480
|
+
ON selection.node_id = reachable.node_id
|
|
11481
|
+
AND selection.object_type_id = reachable.object_type_id
|
|
11482
|
+
LEFT JOIN json_each(selection.property_ids) AS selected_property ON true
|
|
11483
|
+
GROUP BY reachable.project_id, reachable.object_type_id, reachable.primary_id
|
|
11484
|
+
),
|
|
11065
11485
|
_sixb_scope_objects AS (
|
|
11066
|
-
SELECT
|
|
11067
|
-
|
|
11068
|
-
|
|
11069
|
-
|
|
11070
|
-
|
|
11071
|
-
|
|
11072
|
-
|
|
11073
|
-
|
|
11074
|
-
|
|
11075
|
-
|
|
11076
|
-
|
|
11077
|
-
|
|
11078
|
-
AND permission.property_id = property.key
|
|
11079
|
-
),
|
|
11080
|
-
json('{}')
|
|
11081
|
-
) AS properties,
|
|
11082
|
-
stored.created_at,
|
|
11083
|
-
stored.updated_at,
|
|
11084
|
-
stored.version,
|
|
11085
|
-
stored.last_commit_id
|
|
11086
|
-
FROM objects AS stored
|
|
11087
|
-
JOIN _sixb_scope_object_ids AS visible
|
|
11088
|
-
ON visible.project_id = stored.project_id
|
|
11089
|
-
AND visible.object_type_id = stored.object_type_id
|
|
11090
|
-
AND visible.primary_id = stored.primary_id
|
|
11486
|
+
SELECT stored.project_id, stored.object_type_id, stored.primary_id,
|
|
11487
|
+
COALESCE((
|
|
11488
|
+
SELECT json_group_object(property.key, ${sqliteJsonEachValue("property")})
|
|
11489
|
+
FROM json_each(stored.properties) AS property
|
|
11490
|
+
WHERE property.key IN (SELECT value FROM json_each(grants.property_ids))
|
|
11491
|
+
), json('{}')) AS properties,
|
|
11492
|
+
stored.created_at, stored.updated_at, stored.version, stored.last_commit_id
|
|
11493
|
+
FROM _sixb_scope_object_grants AS grants
|
|
11494
|
+
CROSS JOIN objects AS stored
|
|
11495
|
+
ON stored.project_id = grants.project_id
|
|
11496
|
+
AND stored.object_type_id = grants.object_type_id
|
|
11497
|
+
AND stored.primary_id = grants.primary_id
|
|
11091
11498
|
),
|
|
11092
11499
|
_sixb_scope_reached_links(
|
|
11093
11500
|
project_id,
|
|
@@ -11113,16 +11520,16 @@ function compileSelectedReadScopeCte(scopeJson, projectId) {
|
|
|
11113
11520
|
edge.target_id,
|
|
11114
11521
|
selection.property_ids
|
|
11115
11522
|
FROM _sixb_scope_reachable AS parent
|
|
11116
|
-
JOIN _sixb_scope_link_selections AS selection
|
|
11523
|
+
CROSS JOIN _sixb_scope_link_selections AS selection
|
|
11117
11524
|
ON selection.parent_node_id = parent.node_id
|
|
11118
11525
|
AND selection.source_object_type_id = parent.object_type_id
|
|
11119
|
-
JOIN links AS edge
|
|
11526
|
+
CROSS JOIN links AS edge
|
|
11120
11527
|
ON edge.project_id = parent.project_id
|
|
11121
11528
|
AND edge.source_type_id = parent.object_type_id
|
|
11122
11529
|
AND edge.source_id = parent.primary_id
|
|
11123
11530
|
AND edge.link_id = selection.link_id
|
|
11124
11531
|
AND edge.target_type_id = selection.target_object_type_id
|
|
11125
|
-
JOIN objects AS target
|
|
11532
|
+
CROSS JOIN objects AS target
|
|
11126
11533
|
ON target.project_id = edge.project_id
|
|
11127
11534
|
AND target.object_type_id = edge.target_type_id
|
|
11128
11535
|
AND target.primary_id = edge.target_id
|
|
@@ -11294,6 +11701,7 @@ var SQLITE_OBJECT_QUERY_CAPABILITIES = {
|
|
|
11294
11701
|
refs: true,
|
|
11295
11702
|
filter: true,
|
|
11296
11703
|
text: true,
|
|
11704
|
+
vector: true,
|
|
11297
11705
|
sort: true,
|
|
11298
11706
|
limit: true,
|
|
11299
11707
|
page: true,
|
|
@@ -11346,7 +11754,7 @@ var SQLITE_OBJECT_QUERY_CAPABILITIES = {
|
|
|
11346
11754
|
"SQLite object query pushdown supports start/refs/filter/text/sort/limit/page/traverse/set/project/expand over JSON properties and object links.",
|
|
11347
11755
|
"expand hydrates linked objects in-database (top-N per parent via row_number() + json_group_array); core resolves each expansion's cardinality before pushdown, and a mixed/unresolved one stays on the fallback.",
|
|
11348
11756
|
"Ordered decimal predicates and sorting use the bounded core fallback because SQLite has no native exact decimal type; canonical decimal equality remains pushdown-safe.",
|
|
11349
|
-
"Relevance sorting
|
|
11757
|
+
"Relevance sorting and unresolved start.includeSubtypes remain planner fallback or rejection cases."
|
|
11350
11758
|
]
|
|
11351
11759
|
};
|
|
11352
11760
|
|
|
@@ -11369,15 +11777,23 @@ class SqliteObjectStorage {
|
|
|
11369
11777
|
return createSqliteSelectedReader(this.db, params, () => this.queryCapabilities());
|
|
11370
11778
|
}
|
|
11371
11779
|
async queryObjects(params) {
|
|
11780
|
+
if (hasVectorProfile3(params.query))
|
|
11781
|
+
return runDeferredReadTransaction(this.db, () => this.reader.queryObjects(params));
|
|
11372
11782
|
return this.reader.queryObjects(params);
|
|
11373
11783
|
}
|
|
11374
11784
|
async countObjects(params) {
|
|
11785
|
+
if (hasVectorProfile3(params.query))
|
|
11786
|
+
return runDeferredReadTransaction(this.db, () => this.reader.countObjects(params));
|
|
11375
11787
|
return this.reader.countObjects(params);
|
|
11376
11788
|
}
|
|
11377
11789
|
async existsObjects(params) {
|
|
11790
|
+
if (hasVectorProfile3(params.query))
|
|
11791
|
+
return runDeferredReadTransaction(this.db, () => this.reader.existsObjects(params));
|
|
11378
11792
|
return this.reader.existsObjects(params);
|
|
11379
11793
|
}
|
|
11380
11794
|
async facetObjects(params) {
|
|
11795
|
+
if (hasVectorProfile3(params.query))
|
|
11796
|
+
return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params));
|
|
11381
11797
|
return this.reader.facetObjects(params);
|
|
11382
11798
|
}
|
|
11383
11799
|
async getByPrimaryId(params) {
|
|
@@ -11669,6 +12085,7 @@ import {
|
|
|
11669
12085
|
linkRefSortKey,
|
|
11670
12086
|
linkScopeSortKey as linkScopeSortKey2,
|
|
11671
12087
|
MaterializationConflictError as MaterializationConflictError6,
|
|
12088
|
+
MaterializationValidationError as MaterializationValidationError2,
|
|
11672
12089
|
objectRefKey as objectRefKey3,
|
|
11673
12090
|
objectRefSortKey,
|
|
11674
12091
|
telemetryPointKey as telemetryPointKey2,
|
|
@@ -13156,6 +13573,12 @@ class SqliteOntologyMaterializationStorage {
|
|
|
13156
13573
|
this.sessions = new SqliteMaterializationSessions(db, context);
|
|
13157
13574
|
this.writer = new SqliteMaterializationWriter(db);
|
|
13158
13575
|
}
|
|
13576
|
+
assertVectorSession(session, projectId, commitId) {
|
|
13577
|
+
const active = this.sessions.require(session);
|
|
13578
|
+
if (active.header.commit.projectId !== projectId || commitId !== undefined && active.header.commit.id !== commitId) {
|
|
13579
|
+
throw new MaterializationValidationError2("Vector mutation does not belong to this materialization session.");
|
|
13580
|
+
}
|
|
13581
|
+
}
|
|
13159
13582
|
async begin(input) {
|
|
13160
13583
|
assertMaterializationHeader(input);
|
|
13161
13584
|
const session = this.sessions.create(input);
|
|
@@ -13593,7 +14016,7 @@ class SqliteOntologyMaterializationStorage {
|
|
|
13593
14016
|
import { serializeSixbFailure as serializeSixbFailure2 } from "@sixb/core/internal/errors";
|
|
13594
14017
|
import {
|
|
13595
14018
|
MaterializationConflictError as MaterializationConflictError7,
|
|
13596
|
-
MaterializationValidationError as
|
|
14019
|
+
MaterializationValidationError as MaterializationValidationError3
|
|
13597
14020
|
} from "@sixb/core/internal/materialization";
|
|
13598
14021
|
import { ONTOLOGY_OUTBOX_FAILURE_CODES as ONTOLOGY_OUTBOX_FAILURE_CODES2 } from "@sixb/core/storage";
|
|
13599
14022
|
class SqliteOntologyOutboxStorage {
|
|
@@ -13611,22 +14034,28 @@ class SqliteOntologyOutboxStorage {
|
|
|
13611
14034
|
const leaseExpiry = assertTimestamp(input.leaseExpiresAt, "Ontology outbox lease expiry");
|
|
13612
14035
|
assertNonblank(input.leaseId, "Ontology outbox lease id");
|
|
13613
14036
|
if (leaseExpiry <= now) {
|
|
13614
|
-
throw new
|
|
14037
|
+
throw new MaterializationValidationError3("Ontology outbox lease expiry must be later than the claim time.");
|
|
13615
14038
|
}
|
|
14039
|
+
const eligible = this.db.query(`SELECT CAST(rowid AS TEXT) AS rowId
|
|
14040
|
+
FROM ontology_outbox INDEXED BY idx_ontology_outbox_claim
|
|
14041
|
+
WHERE project_id = ? AND published_at IS NULL AND available_at <= ?
|
|
14042
|
+
AND (lease_expires_at IS NULL OR lease_expires_at <= ?)
|
|
14043
|
+
LIMIT ?`).all(input.projectId, input.now, input.now, input.limit + 1);
|
|
14044
|
+
if (eligible.length === 0)
|
|
14045
|
+
return [];
|
|
14046
|
+
const selected = eligible.length <= input.limit ? eligible : this.db.query(`SELECT CAST(rowid AS TEXT) AS rowId
|
|
14047
|
+
FROM ontology_outbox INDEXED BY idx_ontology_outbox_publication_order
|
|
14048
|
+
WHERE project_id = ? AND published_at IS NULL AND available_at <= ?
|
|
14049
|
+
AND (lease_expires_at IS NULL OR lease_expires_at <= ?)
|
|
14050
|
+
ORDER BY created_at, commit_id, commit_ordinal
|
|
14051
|
+
LIMIT ?`).all(input.projectId, input.now, input.now, input.limit);
|
|
13616
14052
|
const rows = this.db.query(`
|
|
13617
14053
|
UPDATE ontology_outbox
|
|
13618
14054
|
SET attempts = attempts + 1, lease_id = ?, lease_expires_at = ?
|
|
13619
|
-
WHERE rowid IN (
|
|
13620
|
-
SELECT rowid
|
|
13621
|
-
FROM ontology_outbox
|
|
13622
|
-
WHERE project_id = ? AND published_at IS NULL AND available_at <= ?
|
|
13623
|
-
AND (lease_expires_at IS NULL OR lease_expires_at <= ?)
|
|
13624
|
-
ORDER BY created_at, commit_id, commit_ordinal
|
|
13625
|
-
LIMIT ?
|
|
13626
|
-
)
|
|
14055
|
+
WHERE rowid IN (SELECT CAST(value AS INTEGER) FROM json_each(?))
|
|
13627
14056
|
RETURNING envelope, available_at, attempts, lease_id, lease_expires_at,
|
|
13628
14057
|
published_at, last_failure, created_at
|
|
13629
|
-
`).all(input.leaseId, input.leaseExpiresAt,
|
|
14058
|
+
`).all(input.leaseId, input.leaseExpiresAt, JSON.stringify(selected.map((row) => row.rowId)));
|
|
13630
14059
|
return rows.map((row) => outboxRecord(row)).sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.envelope.id.localeCompare(right.envelope.id)).map((record) => {
|
|
13631
14060
|
if (record.leaseId === null || record.leaseExpiresAt === null) {
|
|
13632
14061
|
throw new MaterializationConflictError7("outbox-lease", "Ontology outbox claim returned an unpaired lease.");
|
|
@@ -13644,8 +14073,9 @@ class SqliteOntologyOutboxStorage {
|
|
|
13644
14073
|
WITH requested(id) AS MATERIALIZED (
|
|
13645
14074
|
SELECT CAST(value AS TEXT) FROM json_each(?)
|
|
13646
14075
|
), leased(id) AS MATERIALIZED (
|
|
13647
|
-
|
|
13648
|
-
|
|
14076
|
+
-- Drive validation from the bounded ID list even before ANALYZE.
|
|
14077
|
+
SELECT outbox.id FROM requested
|
|
14078
|
+
CROSS JOIN ontology_outbox AS outbox USING (id)
|
|
13649
14079
|
WHERE outbox.project_id = ? AND outbox.lease_id = ?
|
|
13650
14080
|
AND outbox.lease_expires_at IS NOT NULL
|
|
13651
14081
|
), eligible(id) AS (
|
|
@@ -13670,8 +14100,9 @@ class SqliteOntologyOutboxStorage {
|
|
|
13670
14100
|
WITH requested(id) AS MATERIALIZED (
|
|
13671
14101
|
SELECT CAST(value AS TEXT) FROM json_each(?)
|
|
13672
14102
|
), leased(id) AS MATERIALIZED (
|
|
13673
|
-
|
|
13674
|
-
|
|
14103
|
+
-- Drive validation from the bounded ID list even before ANALYZE.
|
|
14104
|
+
SELECT outbox.id FROM requested
|
|
14105
|
+
CROSS JOIN ontology_outbox AS outbox USING (id)
|
|
13675
14106
|
WHERE outbox.project_id = ? AND outbox.lease_id = ?
|
|
13676
14107
|
AND outbox.lease_expires_at IS NOT NULL
|
|
13677
14108
|
), eligible(id) AS (
|
|
@@ -13728,7 +14159,7 @@ function validateLeaseIds(ids) {
|
|
|
13728
14159
|
for (const id of ids) {
|
|
13729
14160
|
assertNonblank(id, "Ontology outbox event id");
|
|
13730
14161
|
if (seen.has(id)) {
|
|
13731
|
-
throw new
|
|
14162
|
+
throw new MaterializationValidationError3(`Ontology outbox lease batch repeats event '${id}'.`);
|
|
13732
14163
|
}
|
|
13733
14164
|
seen.add(id);
|
|
13734
14165
|
}
|
|
@@ -13743,7 +14174,7 @@ function leaseConflict() {
|
|
|
13743
14174
|
}
|
|
13744
14175
|
|
|
13745
14176
|
// src/ontology-storage/sources.ts
|
|
13746
|
-
import { MaterializationValidationError as
|
|
14177
|
+
import { MaterializationValidationError as MaterializationValidationError4 } from "@sixb/core/internal/materialization";
|
|
13747
14178
|
import {
|
|
13748
14179
|
assertSourceBeginInput as assertBeginInput,
|
|
13749
14180
|
assertSourceExecutionIdentity as assertExecutionIdentity,
|
|
@@ -13883,7 +14314,7 @@ class SqliteOntologySourceStorage {
|
|
|
13883
14314
|
throw sourceConflict(`Source materialization '${input.materializationId}' cannot transition from '${manifest.status}' to 'ready'.`);
|
|
13884
14315
|
}
|
|
13885
14316
|
if (input.readyAt < manifest.created_at) {
|
|
13886
|
-
throw new
|
|
14317
|
+
throw new MaterializationValidationError4("Source readyAt cannot precede source createdAt.");
|
|
13887
14318
|
}
|
|
13888
14319
|
this.assertReadyState(manifest, input.rootCount, input.assertionCount);
|
|
13889
14320
|
const result = this.db.query(`
|
|
@@ -13966,7 +14397,7 @@ class SqliteOntologySourceStorage {
|
|
|
13966
14397
|
}
|
|
13967
14398
|
transitionToAbandoned(manifest, abandonedAt) {
|
|
13968
14399
|
if (abandonedAt < manifest.created_at || manifest.ready_at !== null && abandonedAt < manifest.ready_at) {
|
|
13969
|
-
throw new
|
|
14400
|
+
throw new MaterializationValidationError4("Source abandonedAt cannot precede source creation or readiness.");
|
|
13970
14401
|
}
|
|
13971
14402
|
const changed = this.db.query(`
|
|
13972
14403
|
UPDATE ontology_sources
|
|
@@ -13991,7 +14422,7 @@ class SqliteOntologySourceStorage {
|
|
|
13991
14422
|
WHERE project_id = ? AND source_id = ? AND materialization_id = ?
|
|
13992
14423
|
`).get(manifest.project_id, manifest.source_id, manifest.materialization_id, manifest.project_id, manifest.source_id, manifest.materialization_id);
|
|
13993
14424
|
if (counts.assertions !== assertionCount || counts.roots !== rootCount || counts.ordinals !== rootCount || rootCount > 0 && (counts.min_ordinal !== 0 || counts.max_ordinal !== rootCount - 1)) {
|
|
13994
|
-
throw new
|
|
14425
|
+
throw new MaterializationValidationError4("Source ready counts do not match the staged roots and assertions.");
|
|
13995
14426
|
}
|
|
13996
14427
|
assertSourceRootCoverage(this.db, manifest);
|
|
13997
14428
|
const invalid = this.db.query(manifest.projection_kind === "link" ? `
|
|
@@ -14019,7 +14450,7 @@ class SqliteOntologySourceStorage {
|
|
|
14019
14450
|
LIMIT 1
|
|
14020
14451
|
`).get(manifest.project_id, manifest.source_id, manifest.materialization_id);
|
|
14021
14452
|
if (invalid) {
|
|
14022
|
-
throw new
|
|
14453
|
+
throw new MaterializationValidationError4(manifest.projection_kind === "link" ? "Link projection roots must contain exactly their matching link assertion." : "Object projection roots must contain exactly their matching object assertion plus links sourced from that root.");
|
|
14023
14454
|
}
|
|
14024
14455
|
}
|
|
14025
14456
|
findStageRows(input, rows) {
|
|
@@ -14125,16 +14556,228 @@ class SqliteOntologySourceStorage {
|
|
|
14125
14556
|
}
|
|
14126
14557
|
}
|
|
14127
14558
|
|
|
14559
|
+
// src/ontology-storage/vector-indexing.ts
|
|
14560
|
+
import { parseSixbFailure as parseSixbFailure3 } from "@sixb/core/internal/errors";
|
|
14561
|
+
import {
|
|
14562
|
+
VECTOR_INDEXING_FAILURE_CODES
|
|
14563
|
+
} from "@sixb/core/storage";
|
|
14564
|
+
|
|
14565
|
+
class SqliteVectorIndexingStorage {
|
|
14566
|
+
db;
|
|
14567
|
+
run;
|
|
14568
|
+
assertSession;
|
|
14569
|
+
constructor(db, run, assertSession) {
|
|
14570
|
+
this.db = db;
|
|
14571
|
+
this.run = run;
|
|
14572
|
+
this.assertSession = assertSession;
|
|
14573
|
+
}
|
|
14574
|
+
async schedule(input) {
|
|
14575
|
+
this.assertSession(input.session, input.projectId);
|
|
14576
|
+
for (const ref of input.deleted)
|
|
14577
|
+
this.db.query(`DELETE FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ? AND primary_id = ?`).run(input.projectId, ref.objectTypeId, ref.primaryId);
|
|
14578
|
+
const insert = this.db.query(`INSERT INTO object_vector_indexing (project_id, id, object_type_id, primary_id, profile, batch_id, request, status, available_at, dispatch_at)
|
|
14579
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?)
|
|
14580
|
+
ON CONFLICT (project_id, object_type_id, primary_id, profile) DO UPDATE SET id = excluded.id, batch_id = excluded.batch_id, request = excluded.request, status = 'pending', available_at = excluded.available_at, dispatch_at = excluded.dispatch_at, values_json = NULL, error = NULL`);
|
|
14581
|
+
for (const { batchId, ...request } of input.requests)
|
|
14582
|
+
insert.run(input.projectId, request.id, request.ref.objectTypeId, request.ref.primaryId, request.profile, batchId ?? null, JSON.stringify(request), input.availableAt, input.availableAt);
|
|
14583
|
+
}
|
|
14584
|
+
async complete(input) {
|
|
14585
|
+
this.assertSession(input.session, input.projectId);
|
|
14586
|
+
const remove = this.db.query(`DELETE FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ? AND primary_id = ? AND profile = ? AND json_extract(request, '$.configuration') = ? AND json_extract(request, '$.sourceFingerprint') = ?`);
|
|
14587
|
+
for (const entry of input.entries) {
|
|
14588
|
+
remove.run(input.projectId, entry.ref.objectTypeId, entry.ref.primaryId, entry.profile, entry.configuration, entry.sourceFingerprint);
|
|
14589
|
+
}
|
|
14590
|
+
}
|
|
14591
|
+
async dispatched(input) {
|
|
14592
|
+
await this.run(() => {
|
|
14593
|
+
const update = this.db.query(`UPDATE object_vector_indexing SET dispatch_at = ? WHERE project_id = ? AND id = ?`);
|
|
14594
|
+
for (const id of input.ids)
|
|
14595
|
+
update.run(input.nextDispatchAt, input.projectId, id);
|
|
14596
|
+
});
|
|
14597
|
+
}
|
|
14598
|
+
async get(input) {
|
|
14599
|
+
return this.run(() => {
|
|
14600
|
+
const row = this.db.query(`SELECT batch_id, request, status, available_at, values_json, error FROM object_vector_indexing WHERE project_id = ? AND id = ?`).get(input.projectId, input.id);
|
|
14601
|
+
return row ? decode(row) : null;
|
|
14602
|
+
});
|
|
14603
|
+
}
|
|
14604
|
+
async getBatch(input) {
|
|
14605
|
+
return this.run(() => this.db.query(`
|
|
14606
|
+
SELECT batch_id, request, status, available_at, values_json, error FROM object_vector_indexing
|
|
14607
|
+
WHERE project_id = ? AND batch_id = ? ORDER BY id`).all(input.projectId, input.batchId).map(decode));
|
|
14608
|
+
}
|
|
14609
|
+
async getBatchMember(input) {
|
|
14610
|
+
return this.run(() => {
|
|
14611
|
+
const row = this.db.query(`
|
|
14612
|
+
SELECT batch_id, request, status, available_at, values_json, error
|
|
14613
|
+
FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ?
|
|
14614
|
+
AND primary_id = ? AND profile = ? AND batch_id = ?`).get(input.projectId, input.ref.objectTypeId, input.ref.primaryId, input.profile, input.batchId);
|
|
14615
|
+
return row ? decode(row) : null;
|
|
14616
|
+
});
|
|
14617
|
+
}
|
|
14618
|
+
async updateBatch(input) {
|
|
14619
|
+
return this.run(() => this.db.transaction(() => {
|
|
14620
|
+
const status = this.db.query("SELECT status FROM object_vector_indexing WHERE project_id = ? AND id = ?");
|
|
14621
|
+
if (input.requireAll && input.updates.some((update) => status.get(input.projectId, update.id)?.status !== update.expectedStatus))
|
|
14622
|
+
return false;
|
|
14623
|
+
const updateRow = this.db.query(`UPDATE object_vector_indexing
|
|
14624
|
+
SET status = ?, available_at = ?, values_json = COALESCE(?, values_json), error = ?
|
|
14625
|
+
WHERE project_id = ? AND id = ? AND status = ?`);
|
|
14626
|
+
for (const update of input.updates)
|
|
14627
|
+
updateRow.run(update.status, update.availableAt, update.values ? JSON.stringify(update.values) : null, update.error ? JSON.stringify(update.error) : null, input.projectId, update.id, update.expectedStatus);
|
|
14628
|
+
return true;
|
|
14629
|
+
})());
|
|
14630
|
+
}
|
|
14631
|
+
async listDue(input) {
|
|
14632
|
+
return this.run(() => this.db.query(`SELECT batch_id, request, status, available_at, NULL AS values_json, error FROM object_vector_indexing WHERE project_id = ? AND status <> 'failed' AND dispatch_at <= ? ORDER BY dispatch_at, id LIMIT ?`).all(input.projectId, input.now, input.limit).map(decode));
|
|
14633
|
+
}
|
|
14634
|
+
async update(input) {
|
|
14635
|
+
return this.run(() => this.db.query(`UPDATE object_vector_indexing SET status = ?, available_at = ?, values_json = COALESCE(?, values_json), error = ? WHERE project_id = ? AND id = ? AND status = ?`).run(input.status, input.availableAt, input.values ? JSON.stringify(input.values) : null, input.error ? JSON.stringify(input.error) : null, input.projectId, input.id, input.expectedStatus).changes === 1);
|
|
14636
|
+
}
|
|
14637
|
+
async remove(input) {
|
|
14638
|
+
await this.run(() => {
|
|
14639
|
+
this.db.query(`DELETE FROM object_vector_indexing WHERE project_id = ? AND id = ?`).run(input.projectId, input.id);
|
|
14640
|
+
});
|
|
14641
|
+
}
|
|
14642
|
+
}
|
|
14643
|
+
function decode(row) {
|
|
14644
|
+
const request = JSON.parse(row.request);
|
|
14645
|
+
return {
|
|
14646
|
+
...request,
|
|
14647
|
+
...row.batch_id ? { batchId: row.batch_id } : {},
|
|
14648
|
+
status: row.status,
|
|
14649
|
+
availableAt: row.available_at,
|
|
14650
|
+
...row.values_json ? { values: JSON.parse(row.values_json) } : {},
|
|
14651
|
+
...row.error ? {
|
|
14652
|
+
error: parseSixbFailure3(row.error, VECTOR_INDEXING_FAILURE_CODES)
|
|
14653
|
+
} : {}
|
|
14654
|
+
};
|
|
14655
|
+
}
|
|
14656
|
+
|
|
14657
|
+
// src/ontology-storage/vectors.ts
|
|
14658
|
+
import { MaterializationConflictError as MaterializationConflictError8 } from "@sixb/core/internal/materialization";
|
|
14659
|
+
class SqliteOntologyVectorStorage {
|
|
14660
|
+
db;
|
|
14661
|
+
runRootOperation;
|
|
14662
|
+
assertSession;
|
|
14663
|
+
constructor(db, runRootOperation, assertSession) {
|
|
14664
|
+
this.db = db;
|
|
14665
|
+
this.runRootOperation = runRootOperation;
|
|
14666
|
+
this.assertSession = assertSession;
|
|
14667
|
+
}
|
|
14668
|
+
async list(input) {
|
|
14669
|
+
return this.runRootOperation(() => {
|
|
14670
|
+
const rows = this.db.query(`
|
|
14671
|
+
SELECT profile, configuration, source, source_fingerprint, last_commit_id
|
|
14672
|
+
FROM object_vectors
|
|
14673
|
+
WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
|
|
14674
|
+
ORDER BY profile
|
|
14675
|
+
`).all(input.projectId, input.ref.objectTypeId, input.ref.primaryId);
|
|
14676
|
+
return rows.map((row) => ({
|
|
14677
|
+
ref: { ...input.ref },
|
|
14678
|
+
profile: row.profile,
|
|
14679
|
+
configuration: row.configuration,
|
|
14680
|
+
source: JSON.parse(row.source),
|
|
14681
|
+
sourceFingerprint: row.source_fingerprint,
|
|
14682
|
+
lastCommitId: row.last_commit_id
|
|
14683
|
+
}));
|
|
14684
|
+
});
|
|
14685
|
+
}
|
|
14686
|
+
async listBatch(input) {
|
|
14687
|
+
if (!input.refs.length)
|
|
14688
|
+
return [];
|
|
14689
|
+
return this.runRootOperation(() => {
|
|
14690
|
+
const rows = this.db.query(`
|
|
14691
|
+
WITH requested AS (
|
|
14692
|
+
SELECT DISTINCT json_extract(value, '$.objectTypeId') AS object_type_id,
|
|
14693
|
+
json_extract(value, '$.primaryId') AS primary_id FROM json_each(?)
|
|
14694
|
+
)
|
|
14695
|
+
SELECT stored.object_type_id, stored.primary_id, stored.profile, stored.configuration,
|
|
14696
|
+
stored.source, stored.source_fingerprint, stored.last_commit_id
|
|
14697
|
+
FROM requested CROSS JOIN object_vectors AS stored
|
|
14698
|
+
WHERE stored.project_id = ? AND stored.object_type_id = requested.object_type_id
|
|
14699
|
+
AND stored.primary_id = requested.primary_id
|
|
14700
|
+
ORDER BY stored.object_type_id, stored.primary_id, stored.profile
|
|
14701
|
+
`).all(JSON.stringify(input.refs), input.projectId);
|
|
14702
|
+
return rows.map((row) => ({
|
|
14703
|
+
ref: { objectTypeId: row.object_type_id, primaryId: row.primary_id },
|
|
14704
|
+
profile: row.profile,
|
|
14705
|
+
configuration: row.configuration,
|
|
14706
|
+
source: JSON.parse(row.source),
|
|
14707
|
+
sourceFingerprint: row.source_fingerprint,
|
|
14708
|
+
lastCommitId: row.last_commit_id
|
|
14709
|
+
}));
|
|
14710
|
+
});
|
|
14711
|
+
}
|
|
14712
|
+
async removeBatch(input) {
|
|
14713
|
+
this.assertSession(input.session, input.projectId);
|
|
14714
|
+
if (!input.entries.length)
|
|
14715
|
+
return;
|
|
14716
|
+
const { changes } = this.db.query(`
|
|
14717
|
+
DELETE FROM object_vectors WHERE rowid IN (
|
|
14718
|
+
SELECT stored.rowid FROM json_each(?) AS requested CROSS JOIN object_vectors AS stored
|
|
14719
|
+
WHERE stored.project_id = ? AND stored.object_type_id = json_extract(requested.value, '$.ref.objectTypeId')
|
|
14720
|
+
AND stored.primary_id = json_extract(requested.value, '$.ref.primaryId')
|
|
14721
|
+
AND stored.profile = json_extract(requested.value, '$.profile')
|
|
14722
|
+
AND stored.last_commit_id = json_extract(requested.value, '$.expectedCommitId')
|
|
14723
|
+
)
|
|
14724
|
+
`).run(JSON.stringify(input.entries), input.projectId);
|
|
14725
|
+
if (changes !== input.entries.length)
|
|
14726
|
+
throw vectorConflict();
|
|
14727
|
+
}
|
|
14728
|
+
async write(input) {
|
|
14729
|
+
const { value } = input;
|
|
14730
|
+
this.assertSession(input.session, input.projectId, value.lastCommitId);
|
|
14731
|
+
const embedding = encodeSqliteVector(value.values);
|
|
14732
|
+
const changes = input.expectedCommitId === null ? this.db.query(`
|
|
14733
|
+
INSERT INTO object_vectors (
|
|
14734
|
+
project_id, object_type_id, primary_id, profile,
|
|
14735
|
+
configuration, source, source_fingerprint, embedding, last_commit_id
|
|
14736
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
14737
|
+
ON CONFLICT (project_id, object_type_id, primary_id, profile) DO NOTHING
|
|
14738
|
+
`).run(input.projectId, value.ref.objectTypeId, value.ref.primaryId, value.profile, value.configuration, JSON.stringify(value.source), value.sourceFingerprint, embedding, value.lastCommitId).changes : this.db.query(`
|
|
14739
|
+
UPDATE object_vectors
|
|
14740
|
+
SET configuration = ?, source = ?, source_fingerprint = ?, embedding = ?, last_commit_id = ?
|
|
14741
|
+
WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
|
|
14742
|
+
AND profile = ? AND last_commit_id = ?
|
|
14743
|
+
`).run(value.configuration, JSON.stringify(value.source), value.sourceFingerprint, embedding, value.lastCommitId, input.projectId, value.ref.objectTypeId, value.ref.primaryId, value.profile, input.expectedCommitId).changes;
|
|
14744
|
+
if (changes !== 1)
|
|
14745
|
+
throw vectorConflict();
|
|
14746
|
+
}
|
|
14747
|
+
async writeBatch(input) {
|
|
14748
|
+
for (const entry of input.entries) {
|
|
14749
|
+
await this.write({ session: input.session, projectId: input.projectId, ...entry });
|
|
14750
|
+
}
|
|
14751
|
+
}
|
|
14752
|
+
async remove(input) {
|
|
14753
|
+
this.assertSession(input.session, input.projectId);
|
|
14754
|
+
const { changes } = this.db.query(`
|
|
14755
|
+
DELETE FROM object_vectors
|
|
14756
|
+
WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
|
|
14757
|
+
AND profile = ? AND last_commit_id = ?
|
|
14758
|
+
`).run(input.projectId, input.ref.objectTypeId, input.ref.primaryId, input.profile, input.expectedCommitId);
|
|
14759
|
+
if (changes !== 1)
|
|
14760
|
+
throw vectorConflict();
|
|
14761
|
+
}
|
|
14762
|
+
}
|
|
14763
|
+
function vectorConflict() {
|
|
14764
|
+
return new MaterializationConflictError8("effective-state", "Vector changed since it was prepared.");
|
|
14765
|
+
}
|
|
14766
|
+
|
|
14128
14767
|
// src/ontology-storage/index.ts
|
|
14129
14768
|
class SqliteOntologyStorage {
|
|
14130
14769
|
commits;
|
|
14131
14770
|
sources;
|
|
14132
14771
|
materializations;
|
|
14772
|
+
vectorIndexing;
|
|
14773
|
+
vectors;
|
|
14133
14774
|
outbox;
|
|
14134
14775
|
constructor(input) {
|
|
14135
14776
|
this.commits = new SqliteOntologyCommitStorage(input.db, input.runRootOperation);
|
|
14136
14777
|
this.sources = new SqliteOntologySourceStorage(input.db, input.runRootOperation);
|
|
14137
14778
|
this.materializations = new SqliteOntologyMaterializationStorage(input.db, input.transactionContext);
|
|
14779
|
+
this.vectorIndexing = new SqliteVectorIndexingStorage(input.db, input.runRootOperation, (session, projectId) => this.materializations.assertVectorSession(session, projectId));
|
|
14780
|
+
this.vectors = new SqliteOntologyVectorStorage(input.db, input.runRootOperation, (session, projectId, commitId) => this.materializations.assertVectorSession(session, projectId, commitId));
|
|
14138
14781
|
this.outbox = new SqliteOntologyOutboxStorage(input.db, input.runRootOperation);
|
|
14139
14782
|
}
|
|
14140
14783
|
deactivateSessions() {
|
|
@@ -14143,7 +14786,7 @@ class SqliteOntologyStorage {
|
|
|
14143
14786
|
}
|
|
14144
14787
|
|
|
14145
14788
|
// src/pipeline-run-storage.ts
|
|
14146
|
-
import { parseSixbFailure as
|
|
14789
|
+
import { parseSixbFailure as parseSixbFailure4, serializeSixbFailure as serializeSixbFailure3 } from "@sixb/core/internal/errors";
|
|
14147
14790
|
import { assertPipelineRunExecution } from "@sixb/core/internal/pipeline-run-storage-provider";
|
|
14148
14791
|
import { canRequeuePipelineRunAfterEnqueueFailure } from "@sixb/core/internal/storage";
|
|
14149
14792
|
import { PIPELINE_RUN_FAILURE_CODES, PipelineRunError } from "@sixb/core/storage";
|
|
@@ -14465,7 +15108,7 @@ function rowToPipelineRunRecord(row) {
|
|
|
14465
15108
|
datasetId: row.output_dataset_id,
|
|
14466
15109
|
versionId: row.output_version_id
|
|
14467
15110
|
} : undefined,
|
|
14468
|
-
error: row.error === null ? undefined :
|
|
15111
|
+
error: row.error === null ? undefined : parseSixbFailure4(row.error, PIPELINE_RUN_FAILURE_CODES)
|
|
14469
15112
|
};
|
|
14470
15113
|
}
|
|
14471
15114
|
function rowToPipelineStepRunRecord(row) {
|
|
@@ -14486,7 +15129,7 @@ function rowToPipelineStepRunRecord(row) {
|
|
|
14486
15129
|
versionId: row.output_version_id
|
|
14487
15130
|
} : undefined,
|
|
14488
15131
|
rowsWritten: row.rows_written ?? undefined,
|
|
14489
|
-
error: row.error === null ? undefined :
|
|
15132
|
+
error: row.error === null ? undefined : parseSixbFailure4(row.error, PIPELINE_RUN_FAILURE_CODES)
|
|
14490
15133
|
};
|
|
14491
15134
|
}
|
|
14492
15135
|
function assertOptionalNonNegativeInteger(value, fieldName) {
|
|
@@ -15431,7 +16074,7 @@ function invalidRecord2(message) {
|
|
|
15431
16074
|
}
|
|
15432
16075
|
|
|
15433
16076
|
// src/sync-run-storage.ts
|
|
15434
|
-
import { parseSixbFailure as
|
|
16077
|
+
import { parseSixbFailure as parseSixbFailure5, serializeSixbFailure as serializeSixbFailure5 } from "@sixb/core/internal/errors";
|
|
15435
16078
|
import { canRequeueSyncRunAfterEnqueueFailure } from "@sixb/core/internal/storage";
|
|
15436
16079
|
import { assertSyncRunExecution } from "@sixb/core/internal/sync-run-storage-provider";
|
|
15437
16080
|
import { SYNC_RUN_FAILURE_CODES, SyncRunError } from "@sixb/core/storage";
|
|
@@ -15634,7 +16277,7 @@ function rowToSyncRunRecord(row) {
|
|
|
15634
16277
|
} : undefined,
|
|
15635
16278
|
expectedLatestVersionId: row.expected_latest_version_id ?? undefined,
|
|
15636
16279
|
commitMessage: row.commit_message ?? undefined,
|
|
15637
|
-
error: row.error === null ? undefined :
|
|
16280
|
+
error: row.error === null ? undefined : parseSixbFailure5(row.error, SYNC_RUN_FAILURE_CODES),
|
|
15638
16281
|
checkpoint: parseCheckpoint(row.checkpoint)
|
|
15639
16282
|
};
|
|
15640
16283
|
}
|
|
@@ -15775,7 +16418,7 @@ class SqliteTimeseriesStorage {
|
|
|
15775
16418
|
}
|
|
15776
16419
|
|
|
15777
16420
|
// src/webhook-run-storage.ts
|
|
15778
|
-
import { parseSixbFailure as
|
|
16421
|
+
import { parseSixbFailure as parseSixbFailure6, serializeSixbFailure as serializeSixbFailure6 } from "@sixb/core/internal/errors";
|
|
15779
16422
|
import { assertWebhookRunExecution } from "@sixb/core/internal/webhook-run-storage-provider";
|
|
15780
16423
|
import { canRetryWebhookRun, WEBHOOK_RUN_FAILURE_CODES, WebhookRunError } from "@sixb/core/storage";
|
|
15781
16424
|
class SqliteWebhookRunStorage {
|
|
@@ -15921,7 +16564,7 @@ function rowToWebhookRunRecord(row) {
|
|
|
15921
16564
|
requestBodySha256: row.request_body_sha256,
|
|
15922
16565
|
responseStatus: row.response_status ?? undefined,
|
|
15923
16566
|
idempotencyKey: row.idempotency_key ?? undefined,
|
|
15924
|
-
error: row.error === null ? undefined :
|
|
16567
|
+
error: row.error === null ? undefined : parseSixbFailure6(row.error, WEBHOOK_RUN_FAILURE_CODES)
|
|
15925
16568
|
};
|
|
15926
16569
|
}
|
|
15927
16570
|
|
|
@@ -16150,7 +16793,7 @@ function assertNonNegativeInteger(value, fieldName) {
|
|
|
16150
16793
|
}
|
|
16151
16794
|
|
|
16152
16795
|
// src/workflow-run-storage.ts
|
|
16153
|
-
import { parseSixbFailure as
|
|
16796
|
+
import { parseSixbFailure as parseSixbFailure7, serializeSixbFailure as serializeSixbFailure7 } from "@sixb/core/internal/errors";
|
|
16154
16797
|
import {
|
|
16155
16798
|
assertWorkflowAgentNodeRunExecution,
|
|
16156
16799
|
assertWorkflowRunExecution
|
|
@@ -16708,7 +17351,7 @@ function rowToWorkflowRunRecord(row) {
|
|
|
16708
17351
|
queuedAt: row.queued_at ? new Date(row.queued_at) : undefined,
|
|
16709
17352
|
startedAt: new Date(row.started_at),
|
|
16710
17353
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
16711
|
-
error: row.error === null ? undefined :
|
|
17354
|
+
error: row.error === null ? undefined : parseSixbFailure7(row.error, WORKFLOW_RUN_FAILURE_CODES),
|
|
16712
17355
|
attempt: row.attempt,
|
|
16713
17356
|
...row.execution_token && row.execution_queue_lease_expires_at ? {
|
|
16714
17357
|
execution: {
|
|
@@ -16733,7 +17376,7 @@ function rowToWorkflowNodeRunRecord(row) {
|
|
|
16733
17376
|
startedAt: new Date(row.started_at),
|
|
16734
17377
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
16735
17378
|
output: row.output ? parseRecord2(row.output) : undefined,
|
|
16736
|
-
error: row.error === null ? undefined :
|
|
17379
|
+
error: row.error === null ? undefined : parseSixbFailure7(row.error, WORKFLOW_RUN_FAILURE_CODES)
|
|
16737
17380
|
};
|
|
16738
17381
|
}
|
|
16739
17382
|
function assertNonNegativeInteger2(value, fieldName) {
|
|
@@ -16761,7 +17404,7 @@ function rowToWorkflowAgentNodeRunRecord(row) {
|
|
|
16761
17404
|
...row.finish_reason ? { finishReason: row.finish_reason } : {},
|
|
16762
17405
|
...row.trace ? { trace: JSON.parse(row.trace) } : {},
|
|
16763
17406
|
...row.diagnostics ? { diagnostics: JSON.parse(row.diagnostics) } : {},
|
|
16764
|
-
...row.error ? { error:
|
|
17407
|
+
...row.error ? { error: parseSixbFailure7(row.error, AGENT_RUN_FAILURE_CODES3) } : {},
|
|
16765
17408
|
attempt: row.attempt,
|
|
16766
17409
|
...row.execution_token && row.execution_queue_lease_expires_at ? {
|
|
16767
17410
|
execution: {
|
|
@@ -16984,4 +17627,4 @@ export {
|
|
|
16984
17627
|
migrateSqliteStorage
|
|
16985
17628
|
};
|
|
16986
17629
|
|
|
16987
|
-
//# debugId=
|
|
17630
|
+
//# debugId=23F5EFFEE42C282464756E2164756E21
|