@sixb/pg 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +9 -8
  2. package/dist/agents/computers.d.ts +8 -2
  3. package/dist/agents/computers.d.ts.map +1 -1
  4. package/dist/index.js +566 -144
  5. package/dist/index.js.map +20 -17
  6. package/dist/migrations.d.ts.map +1 -1
  7. package/dist/objects/conditional-reader.d.ts +7 -2
  8. package/dist/objects/conditional-reader.d.ts.map +1 -1
  9. package/dist/objects/partitions.d.ts.map +1 -1
  10. package/dist/objects/prepare-queries.d.ts.map +1 -1
  11. package/dist/objects/query-compiler.d.ts +14 -1
  12. package/dist/objects/query-compiler.d.ts.map +1 -1
  13. package/dist/objects/query-indexes.d.ts +13 -1
  14. package/dist/objects/query-indexes.d.ts.map +1 -1
  15. package/dist/objects/read-scope.d.ts.map +1 -1
  16. package/dist/objects/reader.d.ts +14 -1
  17. package/dist/objects/reader.d.ts.map +1 -1
  18. package/dist/objects/storage.d.ts +1 -0
  19. package/dist/objects/storage.d.ts.map +1 -1
  20. package/dist/objects/visibility-indexes.d.ts +22 -0
  21. package/dist/objects/visibility-indexes.d.ts.map +1 -0
  22. package/dist/objects/visibility-statistics.d.ts +26 -0
  23. package/dist/objects/visibility-statistics.d.ts.map +1 -0
  24. package/dist/objects/visibility.d.ts +34 -0
  25. package/dist/objects/visibility.d.ts.map +1 -0
  26. package/dist/pg-client.d.ts.map +1 -1
  27. package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
  28. package/dist/pg-sync-run-storage.d.ts.map +1 -1
  29. package/package.json +3 -3
  30. package/src/agents/computers.ts +54 -5
  31. package/src/migrations/087-agent-computer-background-commands.sql +4 -0
  32. package/src/migrations/088-web-search-model-kind.sql +12 -0
  33. package/src/migrations.ts +6 -0
  34. package/src/objects/conditional-reader.ts +26 -4
  35. package/src/objects/partitions.ts +17 -4
  36. package/src/objects/prepare-queries.ts +8 -3
  37. package/src/objects/query-compiler.ts +64 -23
  38. package/src/objects/query-indexes.ts +93 -27
  39. package/src/objects/read-scope.ts +147 -77
  40. package/src/objects/reader.ts +46 -18
  41. package/src/objects/storage.ts +4 -0
  42. package/src/objects/visibility-indexes.ts +222 -0
  43. package/src/objects/visibility-statistics.ts +73 -0
  44. package/src/objects/visibility.ts +135 -0
  45. package/src/ontology-storage/materializations.ts +3 -1
  46. package/src/pg-client.ts +4 -0
  47. package/src/pg-pipeline-run-storage.ts +5 -7
  48. package/src/pg-sync-run-storage.ts +2 -7
package/dist/index.js CHANGED
@@ -833,7 +833,11 @@ class PgAgentContextCheckpointStore {
833
833
  }
834
834
 
835
835
  // src/agents/computers.ts
836
- import { assertAgentComputerSession } from "@sixb/core/internal/agent-run-storage-provider";
836
+ import {
837
+ AGENT_COMPUTER_BACKGROUND_COMMANDS_MAX,
838
+ assertAgentBackgroundCommandRef,
839
+ assertAgentComputerSession
840
+ } from "@sixb/core/internal/agent-run-storage-provider";
837
841
 
838
842
  class PgAgentComputerSessionStore {
839
843
  sql;
@@ -842,22 +846,54 @@ class PgAgentComputerSessionStore {
842
846
  }
843
847
  async hold(input) {
844
848
  assertAgentComputerSession(input, "SixbPg");
849
+ const background = JSON.stringify(input.backgroundCommands ?? []);
845
850
  await this.sql`
846
851
  INSERT INTO agent_computer_sessions (
847
852
  project_id, owner_principal_type, owner_principal_id, id,
848
- computer_id, computer_name, folder, expires_at
853
+ computer_id, computer_name, folder, expires_at, background_commands
849
854
  ) VALUES (
850
855
  ${input.projectId}, ${input.ownerPrincipal.type}, ${input.ownerPrincipal.id}, ${input.id},
851
- ${input.computer.id}, ${input.computer.name}, ${input.computer.folder}, ${input.expiresAt}
856
+ ${input.computer.id}, ${input.computer.name}, ${input.computer.folder}, ${input.expiresAt},
857
+ ${background}::text::jsonb
852
858
  )
853
859
  ON CONFLICT (project_id, owner_principal_type, owner_principal_id, id)
854
860
  DO UPDATE SET
855
861
  computer_id = excluded.computer_id,
856
862
  computer_name = excluded.computer_name,
857
863
  folder = excluded.folder,
858
- expires_at = excluded.expires_at
864
+ expires_at = excluded.expires_at,
865
+ background_commands = excluded.background_commands
859
866
  `;
860
867
  }
868
+ async requestStop(input) {
869
+ assertAgentBackgroundCommandRef(input.command, "SixbPg");
870
+ const { threadId, id, startedAt } = input.command;
871
+ const ref = JSON.stringify({ threadId, id, startedAt });
872
+ const [row] = await this.sql`
873
+ UPDATE agent_computer_sessions AS session
874
+ SET stop_requests = (
875
+ SELECT COALESCE(jsonb_agg(kept.value ORDER BY kept.position), '[]'::jsonb)
876
+ FROM (
877
+ SELECT request.value, request.position
878
+ FROM jsonb_array_elements(
879
+ CASE
880
+ WHEN session.stop_requests @> jsonb_build_array(${ref}::text::jsonb)
881
+ THEN session.stop_requests
882
+ ELSE session.stop_requests || jsonb_build_array(${ref}::text::jsonb)
883
+ END
884
+ ) WITH ORDINALITY AS request(value, position)
885
+ ORDER BY request.position DESC
886
+ LIMIT ${AGENT_COMPUTER_BACKGROUND_COMMANDS_MAX}
887
+ ) AS kept
888
+ )
889
+ WHERE project_id = ${input.projectId}
890
+ AND owner_principal_type = ${input.ownerPrincipal.type}
891
+ AND owner_principal_id = ${input.ownerPrincipal.id}
892
+ AND id = ${input.id}
893
+ RETURNING *
894
+ `;
895
+ return row ? rowToSession(row) : null;
896
+ }
861
897
  async release(input) {
862
898
  const [row] = await this.sql`
863
899
  DELETE FROM agent_computer_sessions
@@ -895,7 +931,9 @@ function rowToSession(row) {
895
931
  id: row.id,
896
932
  ownerPrincipal: { type: row.owner_principal_type, id: row.owner_principal_id },
897
933
  computer: { id: row.computer_id, name: row.computer_name, folder: row.folder },
898
- expiresAt: new Date(row.expires_at)
934
+ expiresAt: new Date(row.expires_at),
935
+ ...row.background_commands.length > 0 ? { backgroundCommands: row.background_commands } : {},
936
+ ...row.stop_requests.length > 0 ? { stopRequests: row.stop_requests } : {}
899
937
  };
900
938
  }
901
939
 
@@ -10168,6 +10206,28 @@ END $$;
10168
10206
  ANALYZE objects;
10169
10207
  `;
10170
10208
 
10209
+ // src/migrations/087-agent-computer-background-commands.sql
10210
+ var _087_agent_computer_background_commands_default = `-- What each open sixb reports of its threads' background commands, which outlive the turn that
10211
+ -- started them, and the ones someone asked to stop from the web: its renewals hand those back.
10212
+ ALTER TABLE agent_computer_sessions ADD COLUMN background_commands JSONB NOT NULL DEFAULT '[]';
10213
+ ALTER TABLE agent_computer_sessions ADD COLUMN stop_requests JSONB NOT NULL DEFAULT '[]';
10214
+ `;
10215
+
10216
+ // src/migrations/088-web-search-model-kind.sql
10217
+ var _088_web_search_model_kind_default = `ALTER TABLE ai_model_call_usage
10218
+ DROP CONSTRAINT ai_model_call_usage_model_kind_check;
10219
+
10220
+ ALTER TABLE ai_model_call_usage
10221
+ ADD CONSTRAINT ai_model_call_usage_model_kind_check
10222
+ CHECK (
10223
+ model_kind IS NULL
10224
+ OR model_kind IN (
10225
+ 'language', 'image', 'video', 'embedding', 'decision', 'transcription', 'speech', 'reranking',
10226
+ 'web-search'
10227
+ )
10228
+ );
10229
+ `;
10230
+
10171
10231
  // src/migrations.ts
10172
10232
  var POSTGRES_STORAGE_ADAPTER_ID = "SixbPostgresStorage";
10173
10233
  function pgSql(id, sqlText) {
@@ -10420,7 +10480,9 @@ var postgresStorageMigrations = defineMigrations({
10420
10480
  pgSql("083-agent-thread-runs-on", _083_agent_thread_runs_on_default),
10421
10481
  pgSql("084-agent-token-rates", _084_agent_token_rates_default),
10422
10482
  pgSql("085-agent-computer-sessions", _085_agent_computer_sessions_default),
10423
- pgSql("086-partition-objects", _086_partition_objects_default)
10483
+ pgSql("086-partition-objects", _086_partition_objects_default),
10484
+ pgSql("087-agent-computer-background-commands", _087_agent_computer_background_commands_default),
10485
+ pgSql("088-web-search-model-kind", _088_web_search_model_kind_default)
10424
10486
  ]
10425
10487
  });
10426
10488
 
@@ -10445,6 +10507,7 @@ function literal(value) {
10445
10507
  var LOCK_TIMEOUT = "1s";
10446
10508
  var LOCK_ATTEMPTS = 120;
10447
10509
  var LOCK_RETRY_MS = 1000;
10510
+ var ANALYZE_SCALE_FACTOR = "autovacuum_analyze_scale_factor=0.02";
10448
10511
  function objectPartitionName(objectTypeId) {
10449
10512
  return `objects_${createHash3("md5").update(objectTypeId).digest("hex").slice(0, 24)}`;
10450
10513
  }
@@ -10452,9 +10515,14 @@ async function ensureObjectPartition(connection, schema, objectTypeId) {
10452
10515
  const name = objectPartitionName(objectTypeId);
10453
10516
  const partition = `${identifier(schema)}.${identifier(name)}`;
10454
10517
  const [existing] = await connection`
10455
- SELECT to_regclass(${partition})::text AS found`;
10456
- if (existing?.found)
10518
+ SELECT coalesce(reloptions, '{}') @> ARRAY[${ANALYZE_SCALE_FACTOR}] AS tuned
10519
+ FROM pg_class WHERE oid = to_regclass(${partition})`;
10520
+ if (existing) {
10521
+ if (!existing.tuned) {
10522
+ await connection.unsafe(`ALTER TABLE ${partition} SET (${ANALYZE_SCALE_FACTOR})`);
10523
+ }
10457
10524
  return name;
10525
+ }
10458
10526
  for (let attempt = 1;; attempt++) {
10459
10527
  try {
10460
10528
  await createObjectPartition(connection, schema, partition, objectTypeId);
@@ -10498,7 +10566,7 @@ async function createObjectPartition(connection, schema, partition, objectTypeId
10498
10566
  code: "55P03"
10499
10567
  });
10500
10568
  }
10501
- await connection.unsafe(`CREATE TABLE ${partition} (LIKE ${parent} INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE)`);
10569
+ await connection.unsafe(`CREATE TABLE ${partition} (LIKE ${parent} INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE) WITH (${ANALYZE_SCALE_FACTOR})`);
10502
10570
  if (moving)
10503
10571
  await moveRowsFromDefault(connection, parent, fallback, partition, type, references);
10504
10572
  await connection.unsafe(`ALTER TABLE ${parent} ATTACH PARTITION ${partition} FOR VALUES IN (${type})`);
@@ -10555,6 +10623,7 @@ import {
10555
10623
  isVectorProfileQuery,
10556
10624
  linkEndpointObjectTypeIds,
10557
10625
  normalizeVector,
10626
+ sortTieDirection,
10558
10627
  vectorSearchCandidateLimit
10559
10628
  } from "@sixb/core/internal/query";
10560
10629
  var PAGE_TOKEN_PREFIX = "keyset:";
@@ -10567,9 +10636,10 @@ function compilePgObjectQuery(projectId, query, options = {}) {
10567
10636
  let compiled = source.wrapQuery(compileObjectQueryInternal(projectId, query, {
10568
10637
  probeLimit: options.includeTotal === false,
10569
10638
  correlatedTraversal: options.correlatedTraversal,
10639
+ isolateSortOverflow: options.unorderedStart,
10570
10640
  source
10571
10641
  }));
10572
- if (options.sortIndexBound && source === DEFAULT_OBJECT_QUERY_SOURCE) {
10642
+ if (options.sortIndex) {
10573
10643
  let bounded = query;
10574
10644
  const expansions = [];
10575
10645
  while (bounded.kind === "expand") {
@@ -10577,17 +10647,19 @@ function compilePgObjectQuery(projectId, query, options = {}) {
10577
10647
  bounded = bounded.input;
10578
10648
  }
10579
10649
  if (bounded.kind === "page" || bounded.kind === "limit") {
10580
- const branches = [options.sortIndexBound, `NOT (${options.sortIndexBound})`].map((startPredicate, branch) => compileObjectQueryInternal(projectId, bounded, {
10650
+ const { bound, nullRank } = options.sortIndex;
10651
+ const indexed = nullRank ? [`${bound} AND ${nullRank} = 0`, `${bound} AND ${nullRank} = 1`] : [bound];
10652
+ const branches = [...indexed, `NOT (${bound})`].map((startPredicate, branch) => compileObjectQueryInternal(projectId, bounded, {
10581
10653
  probeLimit: options.includeTotal === false,
10582
10654
  source,
10583
10655
  startPredicate,
10584
- isolateSortOverflow: branch === 1,
10656
+ isolateSortOverflow: branch === indexed.length,
10585
10657
  correlatedTraversal: options.correlatedTraversal
10586
10658
  }));
10587
10659
  const limit = bounded.kind === "page" ? bounded.pageSize + 1 : bounded.limit + (options.includeTotal === false ? 1 : 0);
10588
10660
  let rows = {
10589
- sql: `SELECT * FROM ((${branches[0].sql}) UNION ALL (${branches[1].sql})) AS input ORDER BY ${compiled.order.sql} LIMIT ?`,
10590
- args: [...branches[0].args, ...branches[1].args, ...compiled.order.args, limit]
10661
+ sql: `SELECT * FROM (${branches.map((branch) => `(${branch.sql})`).join(" UNION ALL ")}) AS input ORDER BY ${compiled.order.sql} LIMIT ?`,
10662
+ args: [...branches.flatMap((branch) => branch.args), ...compiled.order.args, limit]
10591
10663
  };
10592
10664
  for (const expansion of expansions) {
10593
10665
  rows = expandRows({ ...rows, order: compiled.order }, expansion, {
@@ -10595,7 +10667,7 @@ function compilePgObjectQuery(projectId, query, options = {}) {
10595
10667
  source
10596
10668
  });
10597
10669
  }
10598
- compiled = { ...compiled, ...rows };
10670
+ compiled = { ...compiled, ...source.wrapStatement(rows.sql, rows.args) };
10599
10671
  }
10600
10672
  }
10601
10673
  const traversal = !options.correlatedTraversal && boundedTraversal(query);
@@ -10612,6 +10684,9 @@ function compilePgObjectQuery(projectId, query, options = {}) {
10612
10684
  } : undefined
10613
10685
  };
10614
10686
  }
10687
+ function linksBetweenVisibleObjects(source) {
10688
+ return source.linksBetweenVisibleObjectsTable ?? source.linksTable;
10689
+ }
10615
10690
  function pgObjectsOfType(source, objectTypeId) {
10616
10691
  const typed = source.objectsOfType?.(objectTypeId);
10617
10692
  if (!typed)
@@ -11016,7 +11091,7 @@ function compileTraversalRelation(projectId, input, traversal, source, correlate
11016
11091
  ${endpointPredicate ? `AND (${endpointPredicate})` : ""}
11017
11092
  AND EXISTS (
11018
11093
  SELECT 1 FROM (${input.sql}) AS input
11019
- JOIN ${source.linksTable} AS edge
11094
+ JOIN ${linksBetweenVisibleObjects(source)} AS edge
11020
11095
  ON edge.project_id = input.project_id
11021
11096
  AND edge.${parent}_type_id = input.object_type_id
11022
11097
  AND edge.${parent}_id = input.primary_id
@@ -11059,7 +11134,7 @@ function compileTraversalProbe(projectId, traversal, source) {
11059
11134
  const parent = traversal.direction === "incoming" ? "target" : "source";
11060
11135
  const limit = 1000;
11061
11136
  const statement = compilePgObjectStatement(`SELECT 1 FROM (${input.sql}) AS input
11062
- JOIN ${source.linksTable} AS edge ON edge.project_id = input.project_id
11137
+ JOIN ${linksBetweenVisibleObjects(source)} AS edge ON edge.project_id = input.project_id
11063
11138
  AND edge.${parent}_type_id = input.object_type_id AND edge.${parent}_id = input.primary_id
11064
11139
  WHERE edge.link_id = ? ${traversal.direction === "incoming" && traversal.sourceObjectTypeId ? "AND edge.source_type_id = ?" : ""}
11065
11140
  LIMIT ?`, [
@@ -11131,7 +11206,7 @@ function compileExpansionValue(expansion, parent, path, ctx) {
11131
11206
  whereArgs.push(expansion.sourceObjectTypeId);
11132
11207
  }
11133
11208
  const from = `
11134
- FROM ${ctx.source.linksTable} AS ${edge}
11209
+ FROM ${linksBetweenVisibleObjects(ctx.source)} AS ${edge}
11135
11210
  JOIN ${ctx.source.objectsTable} AS ${target}
11136
11211
  ON ${target}.project_id = ${edge}.project_id
11137
11212
  AND ${target}.object_type_id = ${neighborType}
@@ -11191,7 +11266,8 @@ function compileExpansionOrder(expansion, target, neighborType, neighborId) {
11191
11266
  clauses.push(`CASE WHEN ${jsonTypeExpression(propertyColumn)} IS NULL OR ${jsonTypeExpression(propertyColumn)} = 'null' THEN 1 ELSE 0 END ASC`, `${orderedValueExpression(field.scalarKind, propertyColumn)} ${direction}`);
11192
11267
  args.push(field.propertyId, field.propertyId, field.propertyId);
11193
11268
  }
11194
- clauses.push(`${neighborType} ASC`, `${neighborId} ASC`);
11269
+ const ties = sortTieDirection(expansion.orderBy ?? []).toUpperCase();
11270
+ clauses.push(`${neighborType} ${ties}`, `${neighborId} ${ties}`);
11195
11271
  return { sql: clauses.join(", "), args };
11196
11272
  }
11197
11273
  function compileSet(projectId, op, inputs, ctx) {
@@ -11368,7 +11444,7 @@ function compileAggregateTraversal(projectId, traversal, source) {
11368
11444
  FROM (
11369
11445
  SELECT DISTINCT edge.source_id AS primary_id
11370
11446
  FROM (${input.sql}) AS input
11371
- JOIN ${source.linksTable} AS edge ON edge.project_id = input.project_id
11447
+ JOIN ${linksBetweenVisibleObjects(source)} AS edge ON edge.project_id = input.project_id
11372
11448
  AND edge.target_type_id = input.object_type_id AND edge.target_id = input.primary_id
11373
11449
  WHERE edge.link_id = ? AND edge.source_type_id = ?
11374
11450
  ) AS endpoints
@@ -11695,13 +11771,13 @@ function sortOrderFields(fields) {
11695
11771
  scalarKind: field.scalarKind
11696
11772
  });
11697
11773
  }
11698
- orderFields.push(...identityOrderFields());
11774
+ orderFields.push(...identityOrderFields(sortTieDirection(fields)));
11699
11775
  return orderFields;
11700
11776
  }
11701
- function identityOrderFields() {
11777
+ function identityOrderFields(direction = "asc") {
11702
11778
  return [
11703
- { kind: "column", column: "object_type_id", direction: "asc" },
11704
- { kind: "column", column: "primary_id", direction: "asc" }
11779
+ { kind: "column", column: "object_type_id", direction },
11780
+ { kind: "column", column: "primary_id", direction }
11705
11781
  ];
11706
11782
  }
11707
11783
  function compileKeysetRanges(fields, cursor, propertyColumn) {
@@ -12020,6 +12096,7 @@ async function createQueryIndex(connection, schema, definition) {
12020
12096
  }
12021
12097
  return name;
12022
12098
  }
12099
+ var STATISTICS_KINDS = "mcv";
12023
12100
  async function ensureExpressionStatistics(connection, schema, indexes) {
12024
12101
  const expressionsByType = new Map;
12025
12102
  for (const index of indexes) {
@@ -12032,21 +12109,33 @@ async function ensureExpressionStatistics(connection, schema, indexes) {
12032
12109
  }
12033
12110
  for (const [objectTypeId, expressions] of expressionsByType) {
12034
12111
  const partition = `${identifier(schema)}.${identifier(objectPartitionName(objectTypeId))}`;
12035
- let statisticsChanged = false;
12036
- for (const expression of expressions) {
12037
- const name = `sixb_query_stats_${createHash4("sha256").update(`${objectTypeId}:${expression}`).digest("hex").slice(0, 24)}`;
12038
- const [existing] = await connection`
12039
- SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
12040
- WHERE n.nspname = ${schema} AND s.stxname = ${name}`;
12041
- if (existing)
12042
- continue;
12043
- await connection.unsafe(`CREATE STATISTICS ${identifier(schema)}.${identifier(name)} ON project_id, ${expression} FROM ${partition}`);
12044
- statisticsChanged = true;
12045
- }
12046
- if (statisticsChanged)
12047
- await connection.unsafe(`ANALYZE ${partition}`);
12112
+ await runPgTransactionOn(connection, async () => {
12113
+ let statisticsChanged = false;
12114
+ for (const expression of expressions) {
12115
+ const name = statisticsName(objectTypeId, `${STATISTICS_KINDS}:${expression}`);
12116
+ if (await statisticsExist(connection, schema, name))
12117
+ continue;
12118
+ const superseded = statisticsName(objectTypeId, expression);
12119
+ if (await statisticsExist(connection, schema, superseded)) {
12120
+ await connection.unsafe(`DROP STATISTICS ${identifier(schema)}.${identifier(superseded)}`);
12121
+ }
12122
+ await connection.unsafe(`CREATE STATISTICS ${identifier(schema)}.${identifier(name)} (${STATISTICS_KINDS}) ON project_id, ${expression} FROM ${partition}`);
12123
+ statisticsChanged = true;
12124
+ }
12125
+ if (statisticsChanged)
12126
+ await connection.unsafe(`ANALYZE ${partition}`);
12127
+ });
12048
12128
  }
12049
12129
  }
12130
+ function statisticsName(objectTypeId, definition) {
12131
+ return `sixb_query_stats_${createHash4("sha256").update(`${objectTypeId}:${definition}`).digest("hex").slice(0, 24)}`;
12132
+ }
12133
+ async function statisticsExist(connection, schema, name) {
12134
+ const [existing] = await connection`
12135
+ SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
12136
+ WHERE n.nspname = ${schema} AND s.stxname = ${name}`;
12137
+ return existing !== undefined;
12138
+ }
12050
12139
  function statisticsExpressions(index) {
12051
12140
  const equality = new Set(Object.keys(index.where ?? {}));
12052
12141
  if (index.kind === "filter")
@@ -12109,13 +12198,16 @@ function compileIndex(index) {
12109
12198
  fields.push(`(properties ->> ${literal(propertyId)})`);
12110
12199
  }
12111
12200
  for (const field of index.fields) {
12112
- const property = `properties -> ${literal(field.propertyId)}`;
12113
- fields.push(`(CASE WHEN jsonb_typeof(${property}) IS NULL OR jsonb_typeof(${property}) = 'null' THEN 1 ELSE 0 END)`, `${sortValue(field)} ${field.direction === "desc" ? "DESC" : "ASC"}`);
12201
+ fields.push(sortIndexNullRank(field.propertyId), `${sortValue(field)} ${field.direction === "desc" ? "DESC" : "ASC"}`);
12114
12202
  }
12115
- fields.push("primary_id");
12203
+ fields.push(index.fields.at(-1)?.direction === "desc" ? "primary_id DESC" : "primary_id");
12116
12204
  }
12117
12205
  return `(${fields.join(", ")}) WHERE ${predicate.join(" AND ")}`;
12118
12206
  }
12207
+ function sortIndexNullRank(propertyId) {
12208
+ const property = `properties -> ${literal(propertyId)}`;
12209
+ return `(CASE WHEN jsonb_typeof(${property}) IS NULL OR jsonb_typeof(${property}) = 'null' THEN 1 ELSE 0 END)`;
12210
+ }
12119
12211
  function sortValue(field) {
12120
12212
  const text = `(properties ->> ${literal(field.propertyId)})`;
12121
12213
  if (field.scalarKind === "decimal")
@@ -12177,7 +12269,7 @@ async function readPgSortIndexes(sql) {
12177
12269
  }
12178
12270
  return indexes;
12179
12271
  }
12180
- async function findPgSortIndexBound(sql, catalog, projectId, query) {
12272
+ async function findPgSortIndexRead(sql, catalog, projectId, query) {
12181
12273
  while (query.kind === "expand")
12182
12274
  query = query.input;
12183
12275
  if (query.kind !== "page" && query.kind !== "limit")
@@ -12219,12 +12311,22 @@ async function findPgSortIndexBound(sql, catalog, projectId, query) {
12219
12311
  continue;
12220
12312
  if (index.equality !== undefined && (!Array.isArray(index.equality) || !index.equality.every((property) => typeof property === "string" && equality.has(property))))
12221
12313
  continue;
12222
- if (!index.fields.every((field, i) => field && typeof field.propertyId === "string" && fields[i]?.kind === "property" && fields[i].propertyId === field.propertyId && (field.direction ?? "asc") === (fields[i].direction ?? "asc") && field.scalarKind === "decimal" === (fields[i].scalarKind === "decimal") && isTextScalarKind(field.scalarKind) === isTextScalarKind(fields[i].scalarKind)))
12314
+ if (!index.fields.every((field, i) => field && typeof field.propertyId === "string" && fields[i]?.kind === "property" && fields[i].propertyId === field.propertyId && field.scalarKind === "decimal" === (fields[i].scalarKind === "decimal") && isTextScalarKind(field.scalarKind) === isTextScalarKind(fields[i].scalarKind)))
12223
12315
  continue;
12224
- matches.push(index);
12225
- }
12226
- matches.sort((a, b) => (b.equality?.length ?? 0) - (a.equality?.length ?? 0));
12227
- return matches[0] ? sortIndexBound(matches[0]) : undefined;
12316
+ const forward = index.fields.every((field, i) => (field.direction ?? "asc") === (fields[i]?.direction ?? "asc"));
12317
+ if (forward)
12318
+ matches.push({ index, backwards: false });
12319
+ else if (index.fields.length === 1)
12320
+ matches.push({ index, backwards: true });
12321
+ }
12322
+ matches.sort((a, b) => (b.index.equality?.length ?? 0) - (a.index.equality?.length ?? 0) || Number(a.backwards) - Number(b.backwards));
12323
+ const [match] = matches;
12324
+ if (!match)
12325
+ return;
12326
+ const bound = sortIndexBound(match.index);
12327
+ if (!match.backwards)
12328
+ return { objectTypeId, bound };
12329
+ return { objectTypeId, bound, nullRank: sortIndexNullRank(match.index.fields[0].propertyId) };
12228
12330
  }
12229
12331
  // src/objects/storage.ts
12230
12332
  import { hasVectorProfile as hasVectorProfile4 } from "@sixb/core/internal/query";
@@ -12257,6 +12359,82 @@ class PgAggregateAdmission {
12257
12359
  // src/objects/conditional-reader.ts
12258
12360
  import { hasVectorProfile as hasVectorProfile3 } from "@sixb/core/internal/query";
12259
12361
 
12362
+ // src/objects/visibility.ts
12363
+ function compilePgVisibilityKeys(keys) {
12364
+ switch (keys.kind) {
12365
+ case "everyone":
12366
+ return `CASE WHEN ${predicateSql(keys.when)} THEN ARRAY[${text(EVERYONE)}]::text[] ELSE ${NO_KEYS} END`;
12367
+ case "user": {
12368
+ const ref = propertySql(keys.propertyId);
12369
+ return `CASE WHEN jsonb_typeof(${ref}) = 'object' AND (${ref} - 'type' - 'id') = '{}'::jsonb AND ${ref} -> 'type' = '"user"'::jsonb AND jsonb_typeof(${ref} -> 'id') = 'string' AND char_length(${ref} ->> 'id') <= ${MAX_KEYED_ID_LENGTH} THEN ARRAY[${text(USER_PREFIX)} || ${withoutEscapes(`left(substr((${ref} -> 'id')::text, 2), -1)`)}] ELSE ${NO_KEYS} END`;
12370
+ }
12371
+ case "users": {
12372
+ const users = `@.type == "user" && @.id.type() == "string"`;
12373
+ const keysOf = (filter) => {
12374
+ const path = text(`strict $[*] ? (${filter}).id`);
12375
+ const ids = withoutEscapes(`jsonb_path_query_array(${propertySql(keys.propertyId)}, ${path}, '{}', true)::text`);
12376
+ const separator = `E'\\x1f'`;
12377
+ return `coalesce(string_to_array(${text(USER_PREFIX)} || replace(NULLIF(left(substr(${ids}, 3), -2), ''), '", "', ${separator} || ${text(USER_PREFIX)}), ${separator}), ${NO_KEYS})`;
12378
+ };
12379
+ const bounded = `${users} && @.id like_regex "^.{0,${MAX_KEYED_ID_LENGTH}}$" flag "s"`;
12380
+ return `CASE WHEN octet_length(${propertySql(keys.propertyId)}::text) <= ${SHORT_LIST_BYTES} THEN ${keysOf(users)} ELSE ${keysOf(bounded)} END`;
12381
+ }
12382
+ case "union":
12383
+ return `(${keys.items.map((item) => `(${compilePgVisibilityKeys(item)})`).join(" || ")})`;
12384
+ case "when":
12385
+ return `CASE WHEN ${predicateSql(keys.when)} THEN ${compilePgVisibilityKeys(keys.keys)} ELSE ${NO_KEYS} END`;
12386
+ }
12387
+ }
12388
+ function pgVisibilityReaderKeys(user) {
12389
+ if (!user)
12390
+ return [EVERYONE];
12391
+ const id = JSON.stringify(user.id).slice(1, -1).replaceAll("\\\\", ESCAPED_BACKSLASH).replaceAll("\\\"", ESCAPED_QUOTE);
12392
+ return [EVERYONE, `${USER_PREFIX}${id}`];
12393
+ }
12394
+ function compilePgConditionFilter(condition) {
12395
+ const visibility = condition.visibility;
12396
+ if (!visibility || (visibility.user?.id.length ?? 0) > MAX_KEYED_ID_LENGTH) {
12397
+ return compilePredicate(condition.predicate);
12398
+ }
12399
+ const readerKeys = pgVisibilityReaderKeys(visibility.user);
12400
+ const clauses = visibility.keys.map((keys) => `((${compilePgVisibilityKeys(keys)}) && ?::text[])`);
12401
+ const args = visibility.keys.map(() => readerKeys);
12402
+ if (visibility.remainder) {
12403
+ const remainder = compilePredicate(visibility.remainder);
12404
+ clauses.push(remainder.sql);
12405
+ args.push(...remainder.args);
12406
+ }
12407
+ return { sql: `(${clauses.join(" OR ")})`, args };
12408
+ }
12409
+ var MAX_KEYED_ID_LENGTH = 200;
12410
+ var ESCAPED_BACKSLASH = "\x1E";
12411
+ var ESCAPED_QUOTE = "\x1D";
12412
+ var SHORT_LIST_BYTES = 2000;
12413
+ var EVERYONE = "*";
12414
+ var USER_PREFIX = "u:";
12415
+ var NO_KEYS = "'{}'::text[]";
12416
+ function propertySql(propertyId) {
12417
+ return `(properties -> ${text(propertyId)}::text)`;
12418
+ }
12419
+ function withoutEscapes(printed) {
12420
+ return `replace(replace(${printed}, ${text("\\\\")}, E'\\x1e'), ${text("\\\"")}, E'\\x1d')`;
12421
+ }
12422
+ function text(value) {
12423
+ return literal(value).replace(/\?/g, "\\x3f");
12424
+ }
12425
+ function predicateSql(predicate) {
12426
+ const compiled = compilePredicate(predicate);
12427
+ let index = 0;
12428
+ return `(${compiled.sql.replace(/\?/g, () => inlineValue(compiled.args[index++]))})`;
12429
+ }
12430
+ function inlineValue(value) {
12431
+ if (typeof value === "string")
12432
+ return text(value);
12433
+ if (typeof value === "number" && Number.isFinite(value))
12434
+ return `(${String(value)})`;
12435
+ throw new Error(`[SixbPg] A view condition compares a value it cannot index: ${String(value)}.`);
12436
+ }
12437
+
12260
12438
  // src/objects/read-scope.ts
12261
12439
  function compilePgSelectedObjectReadSource(projectId, scope, maxTraversalFacts) {
12262
12440
  const scopeJson = JSON.stringify({
@@ -12264,8 +12442,9 @@ function compilePgSelectedObjectReadSource(projectId, scope, maxTraversalFacts)
12264
12442
  objects: scope.objects,
12265
12443
  steps: scope.steps
12266
12444
  });
12445
+ const relations = compileSelectedReadScopeCte(scopeJson, projectId);
12267
12446
  const source = {
12268
- ...scopedQuerySource(compileSelectedReadScopeCte(scopeJson, projectId)),
12447
+ ...scopedQuerySource(() => relations),
12269
12448
  objectPropertyPermissionsTable: "_sixb_scope_object_permissions",
12270
12449
  traversalProbe: { sql: "", args: [] }
12271
12450
  };
@@ -12288,56 +12467,84 @@ function compilePgSelectedObjectReadSource(projectId, scope, maxTraversalFacts)
12288
12467
  function compilePgConditionalObjectReadSource(projectId, scope) {
12289
12468
  const filters = new Map(scope.conditions.map((condition) => [
12290
12469
  condition.objectTypeId,
12291
- compilePredicate(condition.predicate)
12470
+ compilePgConditionFilter(condition)
12292
12471
  ]));
12472
+ const relations = compileConditionalReadScopeRelations(projectId, filters);
12293
12473
  return Object.freeze({
12294
- ...scopedQuerySource(compileConditionalReadScopeCte(projectId, filters)),
12474
+ ...scopedQuerySource((sql) => withReadRelations(relations, sql)),
12475
+ linksBetweenVisibleObjectsTable: "links",
12295
12476
  objectsOfType: (objectTypeId) => {
12296
12477
  const filter = filters.get(objectTypeId);
12297
12478
  return filter ? { table: "objects", filter } : { table: "objects" };
12298
12479
  }
12299
12480
  });
12300
12481
  }
12301
- function scopedQuerySource(compiled) {
12302
- const wrapStatement = (sql, args = []) => ({
12303
- sql: `${compiled.sql}
12482
+ function scopedQuerySource(withClause) {
12483
+ const wrapStatement = (sql, args = []) => {
12484
+ const compiled = withClause(sql);
12485
+ return {
12486
+ sql: `${compiled.sql}
12304
12487
  ${sql}`,
12305
- args: [...compiled.args, ...args]
12306
- });
12488
+ args: [...compiled.args, ...args]
12489
+ };
12490
+ };
12307
12491
  return {
12308
12492
  objectsTable: "_sixb_scope_objects",
12309
12493
  linksTable: "_sixb_scope_links",
12310
12494
  wrapStatement,
12311
- wrapQuery: (query) => ({
12312
- ...query,
12313
- ...wrapStatement(query.sql, query.args),
12314
- vectorProbe: query.vectorProbe ? { ...query.vectorProbe, ...wrapStatement(query.vectorProbe.sql, query.vectorProbe.args) } : undefined,
12315
- totalSql: `${compiled.sql}
12316
- ${query.totalSql}`,
12317
- totalArgs: [...compiled.args, ...query.totalArgs],
12318
- ...query.hasMoreProbe ? {
12319
- hasMoreProbe: {
12320
- ...query.hasMoreProbe,
12321
- ...wrapStatement(query.hasMoreProbe.sql, query.hasMoreProbe.args)
12322
- }
12323
- } : {}
12324
- })
12495
+ wrapQuery: (query) => {
12496
+ const total = wrapStatement(query.totalSql, query.totalArgs);
12497
+ return {
12498
+ ...query,
12499
+ ...wrapStatement(query.sql, query.args),
12500
+ vectorProbe: query.vectorProbe ? {
12501
+ ...query.vectorProbe,
12502
+ ...wrapStatement(query.vectorProbe.sql, query.vectorProbe.args)
12503
+ } : undefined,
12504
+ totalSql: total.sql,
12505
+ totalArgs: total.args,
12506
+ ...query.hasMoreProbe ? {
12507
+ hasMoreProbe: {
12508
+ ...query.hasMoreProbe,
12509
+ ...wrapStatement(query.hasMoreProbe.sql, query.hasMoreProbe.args)
12510
+ }
12511
+ } : {}
12512
+ };
12513
+ }
12514
+ };
12515
+ }
12516
+ function withReadRelations(relations, statementSql) {
12517
+ const read = relations.filter((relation) => new RegExp(`\\b${relation.name}\\b`).test(statementSql));
12518
+ if (read.length === 0)
12519
+ return { sql: "", args: [] };
12520
+ return {
12521
+ sql: `WITH ${read.map((relation) => relation.sql).join(`,
12522
+ `)}`,
12523
+ args: read.flatMap((relation) => relation.args)
12325
12524
  };
12326
12525
  }
12327
- function compileConditionalReadScopeCte(projectId, filters) {
12526
+ function compileConditionalReadScopeRelations(projectId, filters) {
12328
12527
  if (filters.size === 0) {
12329
- return {
12330
- sql: `
12331
- WITH
12332
- _sixb_scope_objects AS NOT MATERIALIZED (
12333
- SELECT * FROM objects WHERE project_id = ?
12334
- ),
12335
- _sixb_scope_links AS NOT MATERIALIZED (
12336
- SELECT * FROM links WHERE project_id = ?
12337
- )
12338
- `,
12339
- args: [projectId, projectId]
12340
- };
12528
+ return [
12529
+ {
12530
+ name: "_sixb_scope_objects",
12531
+ sql: `
12532
+ _sixb_scope_objects AS NOT MATERIALIZED (
12533
+ SELECT * FROM objects WHERE project_id = ?
12534
+ )
12535
+ `,
12536
+ args: [projectId]
12537
+ },
12538
+ {
12539
+ name: "_sixb_scope_links",
12540
+ sql: `
12541
+ _sixb_scope_links AS NOT MATERIALIZED (
12542
+ SELECT * FROM links WHERE project_id = ?
12543
+ )
12544
+ `,
12545
+ args: [projectId]
12546
+ }
12547
+ ];
12341
12548
  }
12342
12549
  const conditionedTypeArgs = [...filters.keys()];
12343
12550
  const conditionedTypes = conditionedTypeArgs.map(() => "?::text").join(", ");
@@ -12347,44 +12554,59 @@ function compileConditionalReadScopeCte(projectId, filters) {
12347
12554
  matches.push(`(object_type_id = ?::text AND ${filter.sql})`);
12348
12555
  matchArgs.push(objectTypeId, ...filter.args);
12349
12556
  }
12350
- const visibleEndpoint = (end) => `
12351
- (
12352
- stored.${end}_type_id NOT IN (${conditionedTypes})
12353
- OR EXISTS (
12354
- SELECT 1
12355
- FROM _sixb_scope_objects AS endpoint
12356
- WHERE endpoint.project_id = stored.project_id
12357
- AND endpoint.object_type_id = stored.${end}_type_id
12358
- AND endpoint.primary_id = stored.${end}_id
12359
- )
12360
- )
12361
- `;
12362
- return {
12363
- sql: `
12364
- WITH
12365
- _sixb_scope_objects AS NOT MATERIALIZED (
12366
- SELECT *
12367
- FROM objects
12368
- WHERE project_id = ?
12369
- AND (object_type_id NOT IN (${conditionedTypes}) OR ${matches.join(" OR ")})
12370
- ),
12371
- _sixb_scope_links AS NOT MATERIALIZED (
12372
- SELECT stored.*
12373
- FROM links AS stored
12374
- WHERE stored.project_id = ?
12375
- AND ${visibleEndpoint("source")}
12376
- AND ${visibleEndpoint("target")}
12377
- )
12378
- `,
12379
- args: [
12380
- projectId,
12381
- ...conditionedTypeArgs,
12382
- ...matchArgs,
12383
- projectId,
12384
- ...conditionedTypeArgs,
12385
- ...conditionedTypeArgs
12386
- ]
12557
+ const visibleEndpoint = (end) => {
12558
+ const checks = [];
12559
+ const args = [...conditionedTypeArgs];
12560
+ for (const [objectTypeId, filter] of filters) {
12561
+ checks.push(`
12562
+ (
12563
+ stored.${end}_type_id = ?::text
12564
+ AND EXISTS (
12565
+ SELECT 1
12566
+ FROM objects AS endpoint
12567
+ WHERE endpoint.project_id = stored.project_id
12568
+ AND endpoint.object_type_id = ?::text
12569
+ AND endpoint.primary_id = stored.${end}_id
12570
+ AND (${filter.sql})
12571
+ )
12572
+ )
12573
+ `);
12574
+ args.push(objectTypeId, objectTypeId, ...filter.args);
12575
+ }
12576
+ return {
12577
+ sql: `(stored.${end}_type_id NOT IN (${conditionedTypes}) OR ${checks.join(" OR ")})`,
12578
+ args
12579
+ };
12387
12580
  };
12581
+ const visibleSource = visibleEndpoint("source");
12582
+ const visibleTarget = visibleEndpoint("target");
12583
+ return [
12584
+ {
12585
+ name: "_sixb_scope_objects",
12586
+ sql: `
12587
+ _sixb_scope_objects AS NOT MATERIALIZED (
12588
+ SELECT *
12589
+ FROM objects
12590
+ WHERE project_id = ?
12591
+ AND (object_type_id NOT IN (${conditionedTypes}) OR ${matches.join(" OR ")})
12592
+ )
12593
+ `,
12594
+ args: [projectId, ...conditionedTypeArgs, ...matchArgs]
12595
+ },
12596
+ {
12597
+ name: "_sixb_scope_links",
12598
+ sql: `
12599
+ _sixb_scope_links AS NOT MATERIALIZED (
12600
+ SELECT stored.*
12601
+ FROM links AS stored
12602
+ WHERE stored.project_id = ?
12603
+ AND ${visibleSource.sql}
12604
+ AND ${visibleTarget.sql}
12605
+ )
12606
+ `,
12607
+ args: [projectId, ...visibleSource.args, ...visibleTarget.args]
12608
+ }
12609
+ ];
12388
12610
  }
12389
12611
  function compileSelectedReadScopeCte(scopeJson, projectId) {
12390
12612
  return {
@@ -13010,11 +13232,13 @@ class PgObjectReader {
13010
13232
  source;
13011
13233
  aggregateAdmission;
13012
13234
  sortIndexes;
13013
- constructor(sql, source, aggregateAdmission, sortIndexes) {
13235
+ readsFewObjects;
13236
+ constructor(sql, source, aggregateAdmission, sortIndexes, readsFewObjects) {
13014
13237
  this.sql = sql;
13015
13238
  this.source = source;
13016
13239
  this.aggregateAdmission = aggregateAdmission;
13017
13240
  this.sortIndexes = sortIndexes;
13241
+ this.readsFewObjects = readsFewObjects;
13018
13242
  }
13019
13243
  async assertVectorSearch(params) {
13020
13244
  if (!hasVectorProfile2(params.query))
@@ -13054,9 +13278,13 @@ class PgObjectReader {
13054
13278
  async compilePageQuery(params) {
13055
13279
  const sql = this.sql;
13056
13280
  const source = this.source;
13057
- const sortIndexBound = source === DEFAULT_OBJECT_QUERY_SOURCE && this.sortIndexes ? await findPgSortIndexBound(sql, this.sortIndexes, params.projectId, params.query) : undefined;
13281
+ let sortIndex = this.sortIndexes ? await findPgSortIndexRead(sql, this.sortIndexes, params.projectId, params.query) : undefined;
13282
+ const unorderedStart = sortIndex !== undefined && await this.readsFewObjects?.(sql, sortIndex.objectTypeId) === true;
13283
+ if (unorderedStart)
13284
+ sortIndex = undefined;
13058
13285
  let compiled = compilePgObjectQuery(params.projectId, params.query, {
13059
- sortIndexBound,
13286
+ sortIndex,
13287
+ unorderedStart,
13060
13288
  includeTotal: params.includeTotal,
13061
13289
  source
13062
13290
  });
@@ -13068,7 +13296,8 @@ class PgObjectReader {
13068
13296
  includeTotal: params.includeTotal,
13069
13297
  source,
13070
13298
  correlatedTraversal: true,
13071
- sortIndexBound
13299
+ sortIndex,
13300
+ unorderedStart
13072
13301
  });
13073
13302
  }
13074
13303
  }
@@ -13231,16 +13460,29 @@ class PgObjectReader {
13231
13460
  const result = new Map;
13232
13461
  if (params.items.length === 0)
13233
13462
  return result;
13234
- const items = params.items.map((item, batchIndex) => ({ ...item, batchIndex }));
13235
- const statement = compilePgObjectStatement(`
13463
+ const requestedByType = new Map;
13464
+ for (const [batchIndex, item] of params.items.entries()) {
13465
+ const requested = requestedByType.get(item.objectTypeId) ?? [];
13466
+ requested.push({ primaryId: item.primaryId, batchIndex });
13467
+ requestedByType.set(item.objectTypeId, requested);
13468
+ }
13469
+ const branches = [];
13470
+ const args = [];
13471
+ for (const [objectTypeId, requested] of requestedByType) {
13472
+ const objects = pgObjectsOfType(source, objectTypeId);
13473
+ branches.push(`
13236
13474
  SELECT object.*, requested."batchIndex" AS _batch_index
13237
13475
  FROM jsonb_to_recordset(?::text::jsonb)
13238
- AS requested("objectTypeId" text, "primaryId" text, "batchIndex" integer)
13239
- JOIN ${source.objectsTable} AS object
13476
+ AS requested("primaryId" text, "batchIndex" integer)
13477
+ JOIN ${objects.table} AS object
13240
13478
  ON object.project_id = ?
13241
- AND object.object_type_id = requested."objectTypeId"
13479
+ AND object.object_type_id = ?::text
13242
13480
  AND object.primary_id = requested."primaryId"
13243
- `, [JSON.stringify(items), params.projectId], source);
13481
+ ${objects.filter}
13482
+ `);
13483
+ args.push(JSON.stringify(requested), params.projectId, objectTypeId, ...objects.args);
13484
+ }
13485
+ const statement = compilePgObjectStatement(branches.join(" UNION ALL "), args, source);
13244
13486
  const rows = await sql.unsafe(statement.sql, statement.args);
13245
13487
  const rowsByIndex = new Map(rows.map((row) => [row._batch_index, row]));
13246
13488
  for (const [index, item] of params.items.entries()) {
@@ -13592,13 +13834,186 @@ function createPgScopedReader(params, queryCapabilities) {
13592
13834
  });
13593
13835
  }
13594
13836
 
13837
+ // src/objects/visibility-indexes.ts
13838
+ import { createHash as createHash6 } from "crypto";
13839
+ async function preparePgVisibilityIndexes(connection, schema, plan) {
13840
+ if (plan.objectTypes.every((type) => type.visibility === undefined)) {
13841
+ return { names: [], warnings: [] };
13842
+ }
13843
+ const warnings = [];
13844
+ const built = [];
13845
+ for (const index of visibilityIndexes(plan)) {
13846
+ const warning = await createVisibilityIndex(connection, schema, plan.projectId, index);
13847
+ if (warning)
13848
+ warnings.push(warning);
13849
+ else
13850
+ built.push(index);
13851
+ }
13852
+ await ensureVisibilityStatistics(connection, schema, built);
13853
+ await dropUnusedVisibilityIndexes(connection, schema, plan.projectId, built);
13854
+ return { names: built.map((index) => index.name), warnings };
13855
+ }
13856
+ function visibilityIndexes(plan) {
13857
+ return plan.objectTypes.flatMap((type) => {
13858
+ const partition = objectPartitionName(type.objectTypeId);
13859
+ return (type.visibility ?? []).map((keys) => {
13860
+ const expression = compilePgVisibilityKeys(keys);
13861
+ const statistics = visibilityStatisticsName(type.objectTypeId, keys);
13862
+ const name = `sixb_visibility_${digest(`${partition}:${plan.projectId}:${expression}`, 32)}`;
13863
+ return { objectTypeId: type.objectTypeId, partition, expression, name, statistics };
13864
+ });
13865
+ });
13866
+ }
13867
+ function visibilityStatisticsName(objectTypeId, keys) {
13868
+ const expression = compilePgVisibilityKeys(keys);
13869
+ return `sixb_visibility_stats_${digest(`${objectPartitionName(objectTypeId)}:${expression}`, 24)}`;
13870
+ }
13871
+ async function createVisibilityIndex(connection, schema, projectId, index) {
13872
+ const table = `${identifier(schema)}.${identifier(index.partition)}`;
13873
+ try {
13874
+ await connection.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(index.name)} ON ${table} USING gin ((${index.expression})) WHERE project_id = ${literal(projectId)}`);
13875
+ } catch (error) {
13876
+ if (error.code !== FUNCTION_NOT_IMMUTABLE)
13877
+ throw error;
13878
+ return conditionNotIndexed(index.objectTypeId, error);
13879
+ }
13880
+ const [built] = await connection`
13881
+ SELECT i.indisvalid AS valid FROM pg_index i
13882
+ JOIN pg_class c ON c.oid = i.indexrelid
13883
+ JOIN pg_namespace n ON n.oid = c.relnamespace
13884
+ WHERE n.nspname = ${schema} AND c.relname = ${index.name}`;
13885
+ if (!built?.valid) {
13886
+ throw new Error(`[SixbPg] Visibility index ${schema}.${index.name} is invalid after an interrupted build. Drop that index concurrently and prepare again.`);
13887
+ }
13888
+ const comment = {
13889
+ kind: "visibility",
13890
+ projectId,
13891
+ statistics: index.statistics
13892
+ };
13893
+ await connection.unsafe(`COMMENT ON INDEX ${identifier(schema)}.${identifier(index.name)} IS ${literal(JSON.stringify(comment))}`);
13894
+ return;
13895
+ }
13896
+ var FUNCTION_NOT_IMMUTABLE = "42P17";
13897
+ async function ensureVisibilityStatistics(connection, schema, indexes) {
13898
+ const partitions = new Map;
13899
+ for (const index of indexes) {
13900
+ partitions.set(index.partition, [...partitions.get(index.partition) ?? [], index]);
13901
+ }
13902
+ for (const [partition, partitionIndexes] of partitions) {
13903
+ const table = `${identifier(schema)}.${identifier(partition)}`;
13904
+ await runPgTransactionOn(connection, async () => {
13905
+ let created = false;
13906
+ for (const index of partitionIndexes) {
13907
+ const [existing] = await connection`
13908
+ SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
13909
+ WHERE n.nspname = ${schema} AND s.stxname = ${index.statistics}`;
13910
+ if (existing)
13911
+ continue;
13912
+ await connection.unsafe(`CREATE STATISTICS ${identifier(schema)}.${identifier(index.statistics)} ON (${index.expression}) FROM ${table}`);
13913
+ created = true;
13914
+ }
13915
+ if (created)
13916
+ await connection.unsafe(`ANALYZE ${table}`);
13917
+ });
13918
+ }
13919
+ }
13920
+ async function dropUnusedVisibilityIndexes(connection, schema, projectId, current) {
13921
+ const currentNames = new Set(current.map((index) => index.name));
13922
+ const existing = await readVisibilityIndexes(connection, schema);
13923
+ const unused = existing.filter((index) => index.comment.projectId === projectId && !currentNames.has(index.name));
13924
+ for (const index of unused) {
13925
+ await connection.unsafe(`DROP INDEX CONCURRENTLY IF EXISTS ${identifier(schema)}.${identifier(index.name)}`);
13926
+ }
13927
+ const kept = existing.filter((index) => !unused.includes(index));
13928
+ for (const statistics of new Set(unused.map((index) => index.comment.statistics))) {
13929
+ if (kept.some((index) => index.comment.statistics === statistics))
13930
+ continue;
13931
+ await connection.unsafe(`DROP STATISTICS IF EXISTS ${identifier(schema)}.${identifier(statistics)}`);
13932
+ }
13933
+ }
13934
+ async function readVisibilityIndexes(connection, schema) {
13935
+ const rows = await connection`
13936
+ SELECT c.relname AS name, d.description AS definition
13937
+ FROM pg_index i
13938
+ JOIN pg_class c ON c.oid = i.indexrelid
13939
+ JOIN pg_namespace n ON n.oid = c.relnamespace
13940
+ JOIN pg_description d
13941
+ ON d.objoid = c.oid AND d.classoid = 'pg_class'::regclass AND d.objsubid = 0
13942
+ WHERE n.nspname = ${schema} AND c.relname LIKE 'sixb_visibility_%'`;
13943
+ return rows.flatMap((row) => {
13944
+ const comment = parseComment(row.definition);
13945
+ return comment ? [{ name: row.name, comment }] : [];
13946
+ });
13947
+ }
13948
+ function parseComment(definition) {
13949
+ try {
13950
+ const comment = JSON.parse(definition);
13951
+ return comment?.kind === "visibility" && typeof comment.projectId === "string" && typeof comment.statistics === "string" ? comment : undefined;
13952
+ } catch {
13953
+ return;
13954
+ }
13955
+ }
13956
+ function conditionNotIndexed(objectTypeId, error) {
13957
+ const reason = error instanceof Error ? error.message : String(error);
13958
+ return `${objectTypeId}: a view condition cannot be indexed (${reason}); members read the type by checking each object.`;
13959
+ }
13960
+ function digest(value, length) {
13961
+ return createHash6("sha256").update(value).digest("hex").slice(0, length);
13962
+ }
13963
+
13964
+ // src/objects/visibility-statistics.ts
13965
+ var VISIBILITY_STATISTICS_TTL_MS = 60000;
13966
+
13967
+ class PgVisibilityStatisticsCatalog {
13968
+ #read;
13969
+ commonKeys(sql) {
13970
+ const now = Date.now();
13971
+ if (this.#read && now - this.#read.at < VISIBILITY_STATISTICS_TTL_MS)
13972
+ return this.#read.keys;
13973
+ const read = { at: now, keys: readCommonKeys(sql) };
13974
+ this.#read = read;
13975
+ read.keys.catch(() => {
13976
+ if (this.#read === read)
13977
+ this.#read = undefined;
13978
+ });
13979
+ return read.keys;
13980
+ }
13981
+ }
13982
+ async function readCommonKeys(sql) {
13983
+ const rows = await sql`
13984
+ SELECT statistics_name AS name, most_common_elems::text::text[] AS keys
13985
+ FROM pg_stats_ext_exprs
13986
+ WHERE schemaname = current_schema() AND statistics_name LIKE 'sixb\\_visibility\\_stats\\_%'`;
13987
+ return new Map(rows.map((row) => [row.name, new Set(row.keys ?? [])]));
13988
+ }
13989
+ function readsFewObjects(condition, common) {
13990
+ const visibility = condition.visibility;
13991
+ if (!visibility || visibility.remainder)
13992
+ return false;
13993
+ if ((visibility.user?.id.length ?? 0) > MAX_KEYED_ID_LENGTH)
13994
+ return false;
13995
+ const readerKeys = pgVisibilityReaderKeys(visibility.user);
13996
+ for (const keys of visibility.keys) {
13997
+ const listed = common.get(visibilityStatisticsName(condition.objectTypeId, keys));
13998
+ if (!listed || readerKeys.some((key) => listed.has(key)))
13999
+ return false;
14000
+ }
14001
+ return true;
14002
+ }
14003
+
13595
14004
  // src/objects/conditional-reader.ts
13596
14005
  function createPgConditionalReader(sql, params, options) {
13597
14006
  const source = compilePgConditionalObjectReadSource(params.projectId, params.scope);
13598
- const reader = new PgObjectReader(sql, source, options.aggregateAdmission);
14007
+ const fewObjects = async (client, objectTypeId) => {
14008
+ const condition = params.scope.conditions.find((item) => item.objectTypeId === objectTypeId);
14009
+ if (!condition)
14010
+ return false;
14011
+ return readsFewObjects(condition, await options.visibilityStatistics.commonKeys(client));
14012
+ };
14013
+ const reader = new PgObjectReader(sql, source, options.aggregateAdmission, options.sortIndexes, fewObjects);
13599
14014
  return createPgScopedReader({
13600
14015
  projectId: params.projectId,
13601
- read: (run, query) => query && hasVectorProfile3(query) ? runPgRepeatableReadTransaction(sql, (tx) => run(new PgObjectReader(tx, source))) : run(reader)
14016
+ read: (run, query) => query && hasVectorProfile3(query) ? runPgRepeatableReadTransaction(sql, (tx) => run(new PgObjectReader(tx, source, undefined, options.sortIndexes, fewObjects))) : run(reader)
13602
14017
  }, options.queryCapabilities);
13603
14018
  }
13604
14019
 
@@ -13699,6 +14114,7 @@ class PgObjectStorage {
13699
14114
  sortIndexes;
13700
14115
  aggregateAdmission;
13701
14116
  reader;
14117
+ visibilityStatistics = new PgVisibilityStatisticsCatalog;
13702
14118
  constructor(sql, sortIndexes = new PgSortIndexCatalog, maxConcurrentAggregates = 1) {
13703
14119
  this.sql = sql;
13704
14120
  this.sortIndexes = sortIndexes;
@@ -13714,6 +14130,8 @@ class PgObjectStorage {
13714
14130
  createConditionalReadScope(params) {
13715
14131
  return createPgConditionalReader(this.sql, params, {
13716
14132
  aggregateAdmission: this.aggregateAdmission,
14133
+ sortIndexes: this.sortIndexes,
14134
+ visibilityStatistics: this.visibilityStatistics,
13717
14135
  queryCapabilities: () => this.queryCapabilities()
13718
14136
  });
13719
14137
  }
@@ -13816,7 +14234,7 @@ function assertReconciliationPageLimit(limit) {
13816
14234
  }
13817
14235
  }
13818
14236
  // src/objects/prepare-queries.ts
13819
- import { createHash as createHash6 } from "crypto";
14237
+ import { createHash as createHash7 } from "crypto";
13820
14238
  async function preparePgObjectQueries(sql, schema, plan) {
13821
14239
  const fingerprint = preparationFingerprint(plan);
13822
14240
  const current = await readCompletedPreparation(sql, fingerprint);
@@ -13849,10 +14267,11 @@ function preparationFingerprint(plan) {
13849
14267
  objectTypeId: type.objectTypeId,
13850
14268
  properties: [...type.properties].sort((a, b) => a.propertyId.localeCompare(b.propertyId)),
13851
14269
  indexes: type.indexes.map(normalizeIndex).sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
13852
- links: [...type.links].sort((a, b) => a.linkId.localeCompare(b.linkId))
14270
+ links: [...type.links].sort((a, b) => a.linkId.localeCompare(b.linkId)),
14271
+ ...type.visibility ? { visibility: type.visibility.map(compilePgVisibilityKeys) } : {}
13853
14272
  })).sort((a, b) => a.objectTypeId.localeCompare(b.objectTypeId));
13854
- const definition = { version: 3, projectId: plan.projectId, objectTypes };
13855
- return createHash6("sha256").update(JSON.stringify(definition)).digest("hex");
14273
+ const definition = { version: 5, projectId: plan.projectId, objectTypes };
14274
+ return createHash7("sha256").update(JSON.stringify(definition)).digest("hex");
13856
14275
  }
13857
14276
  function normalizeIndex(index) {
13858
14277
  if (index.kind === "sort") {
@@ -13889,11 +14308,12 @@ async function prepareStructures(sql, schema, plan) {
13889
14308
  await ensureObjectPartition(sql, schema, type.objectTypeId);
13890
14309
  const names = await ensurePgObjectQueryIndexes(sql, schema, indexes);
13891
14310
  const texts = await preparePgObjectTextCounts(sql, schema, textCounts);
14311
+ const visibility = await preparePgVisibilityIndexes(sql, schema, plan);
13892
14312
  return {
13893
14313
  status: "prepared",
13894
14314
  objectTypes: plan.objectTypes.length,
13895
- indexes: new Set([...names, ...texts]).size,
13896
- warnings
14315
+ indexes: new Set([...names, ...texts, ...visibility.names]).size,
14316
+ warnings: [...warnings, ...visibility.warnings]
13897
14317
  };
13898
14318
  }
13899
14319
  async function linkWarnings(sql, plan) {
@@ -21425,7 +21845,8 @@ var DEFAULT_CONNECT_TIMEOUT_SECONDS = 10;
21425
21845
  function createPgClient(options) {
21426
21846
  const connection = {
21427
21847
  search_path: options.schemaName,
21428
- enable_incremental_sort: "off"
21848
+ enable_incremental_sort: "off",
21849
+ jit: "off"
21429
21850
  };
21430
21851
  if (options.statementTimeoutMillis !== undefined) {
21431
21852
  connection.statement_timeout = options.statementTimeoutMillis;
@@ -21720,7 +22141,10 @@ function jsonText(value) {
21720
22141
  // src/pg-pipeline-run-storage.ts
21721
22142
  import { parseSixbFailure as parseSixbFailure4, serializeSixbFailure as serializeSixbFailure3 } from "@sixb/core/internal/errors";
21722
22143
  import { assertPipelineRunExecution } from "@sixb/core/internal/pipeline-run-storage-provider";
21723
- import { canRequeuePipelineRunAfterEnqueueFailure } from "@sixb/core/internal/storage";
22144
+ import {
22145
+ canFinishPipelineRun,
22146
+ canRequeuePipelineRunAfterEnqueueFailure
22147
+ } from "@sixb/core/internal/storage";
21724
22148
  import { PIPELINE_RUN_FAILURE_CODES, PipelineRunError } from "@sixb/core/storage";
21725
22149
 
21726
22150
  // src/latest-run-query.ts
@@ -21841,8 +22265,7 @@ class PgPipelineRunStorage {
21841
22265
  if (!existing) {
21842
22266
  throw new PipelineRunError(`[SixbPg] Pipeline run '${input.id}' not found for project '${input.projectId}'.`);
21843
22267
  }
21844
- const enqueueFailure = input.status === "failed" && input.error.code === "queue.enqueue_failed";
21845
- if (enqueueFailure && existing.status !== "queued" || !enqueueFailure && existing.status !== "running") {
22268
+ if (!canFinishPipelineRun(existing.status, input)) {
21846
22269
  throw new PipelineRunError(`[SixbPg] Pipeline run '${input.id}' cannot finish from status '${existing.status}'.`);
21847
22270
  }
21848
22271
  const [updated] = input.status === "succeeded" ? await tx`
@@ -22873,7 +23296,7 @@ function toIsoString3(value) {
22873
23296
 
22874
23297
  // src/pg-sync-run-storage.ts
22875
23298
  import { parseSixbFailure as parseSixbFailure5, serializeSixbFailure as serializeSixbFailure5 } from "@sixb/core/internal/errors";
22876
- import { canRequeueSyncRunAfterEnqueueFailure } from "@sixb/core/internal/storage";
23299
+ import { canFinishSyncRun, canRequeueSyncRunAfterEnqueueFailure } from "@sixb/core/internal/storage";
22877
23300
  import { assertSyncRunExecution } from "@sixb/core/internal/sync-run-storage-provider";
22878
23301
  import { SYNC_RUN_FAILURE_CODES, SyncRunError } from "@sixb/core/storage";
22879
23302
  class PgSyncRunStorage {
@@ -22974,8 +23397,7 @@ class PgSyncRunStorage {
22974
23397
  if (!existing) {
22975
23398
  throw new SyncRunError(`[SixbPg] Sync run '${input.id}' not found for project '${input.projectId}'.`);
22976
23399
  }
22977
- const enqueueFailure = input.status === "failed" && input.error.code === "queue.enqueue_failed";
22978
- if (enqueueFailure && existing.status !== "queued" || !enqueueFailure && existing.status !== "running") {
23400
+ if (!canFinishSyncRun(existing.status, input)) {
22979
23401
  throw new SyncRunError(`[SixbPg] Sync run '${input.id}' cannot finish from status '${existing.status}'.`);
22980
23402
  }
22981
23403
  if (input.status === "succeeded") {
@@ -24914,4 +25336,4 @@ export {
24914
25336
  migratePostgresStorage
24915
25337
  };
24916
25338
 
24917
- //# debugId=09BC0C242FA6748C64756E2164756E21
25339
+ //# debugId=8C7AEC74F77610B564756E2164756E21