@sixb/pg 0.1.21 → 0.1.22

package/src/objects/scoped-reader.ts

@@ -0,0 +1,101 @@
+import type { ObjectQuery } from "@sixb/core"
+import type {
+  ObjectQueryCapabilities,
+  ObjectReadExecutionLimits,
+  ObjectReadStorage,
+} from "@sixb/core/storage"
+import {
+  assertObjectReaderProject,
+  assertObjectReadFacetCount,
+  assertObjectReadOutputWithinLimit,
+} from "@sixb/core/storage"
+import { assertLinkQueryLimit, type PgObjectReader } from "./reader"
+
+/** Run one terminal on a reader over the scope's source; a query terminal passes its query. */
+export type PgScopedRead = <T>(
+  run: (reader: PgObjectReader) => Promise<T>,
+  query?: ObjectQuery
+) => Promise<T>
+
+/** Answer every terminal through one scoped source; the scope decides how each read runs. */
+export function createPgScopedReader(
+  params: {
+    projectId: string
+    read: PgScopedRead
+    limits?: ObjectReadExecutionLimits
+  },
+  queryCapabilities: () => ObjectQueryCapabilities
+): ObjectReadStorage {
+  const { projectId, limits } = params
+  const assertProject = (actualProjectId: string): void =>
+    assertObjectReaderProject(projectId, actualProjectId)
+  const read = async <T>(
+    run: (reader: PgObjectReader) => Promise<T>,
+    query?: ObjectQuery
+  ): Promise<T> => {
+    const value = await params.read(run, query)
+    if (limits) assertObjectReadOutputWithinLimit(value, limits)
+    return value
+  }
+  const readMap = async <TKey, TValue>(
+    run: (reader: PgObjectReader) => Promise<Map<TKey, TValue>>
+  ): Promise<Map<TKey, TValue>> => {
+    const value = await params.read(run)
+    if (limits) assertObjectReadOutputWithinLimit([...value.entries()], limits)
+    return value
+  }
+
+  return Object.freeze({
+    queryCapabilities,
+    queryObjects: async (input) => {
+      assertProject(input.projectId)
+      return read((reader) => reader.queryObjects(input), input.query)
+    },
+    countObjects: async (input) => {
+      assertProject(input.projectId)
+      return read((reader) => reader.countObjects(input), input.query)
+    },
+    existsObjects: async (input) => {
+      assertProject(input.projectId)
+      return read((reader) => reader.existsObjects(input), input.query)
+    },
+    facetObjects: async (input) => {
+      assertProject(input.projectId)
+      assertObjectReadFacetCount(input.facets.length)
+      return read((reader) => reader.facetObjects(input), input.query)
+    },
+    getByPrimaryId: async (input) => {
+      assertProject(input.projectId)
+      return read((reader) => reader.getByPrimaryId(input))
+    },
+    getByUniqueValue: async (input) => {
+      assertProject(input.projectId)
+      return read((reader) => reader.getByUniqueValue(input))
+    },
+    selectsObjectProperties: async (input) => {
+      assertProject(input.projectId)
+      return read((reader) => reader.selectsObjectProperties(input))
+    },
+    listLinks: async (input) => {
+      assertProject(input.projectId)
+      return read((reader) => reader.listLinks(input))
+    },
+    getByPrimaryIdBatch: async (input) => {
+      assertProject(input.projectId)
+      return readMap((reader) => reader.getByPrimaryIdBatch(input))
+    },
+    listLinksBatch: async (input) => {
+      assertProject(input.projectId)
+      return readMap((reader) => reader.listLinksBatch(input))
+    },
+    queryLinks: async (input) => {
+      assertProject(input.projectId)
+      assertLinkQueryLimit(input.limit)
+      return read((reader) => reader.queryLinks(input))
+    },
+    list: async (input) => {
+      assertProject(input.projectId)
+      return read((reader) => reader.list(input))
+    },
+  } satisfies ObjectReadStorage)
+}

package/src/objects/selected-reader.ts

@@ -4,19 +4,14 @@ import type {
   ObjectReadExecutionLimits,
   ObjectReadStorage,
 } from "@sixb/core/storage"
-import {
-  assertObjectReaderProject,
-  assertObjectReadFacetCount,
-  assertObjectReadOutputWithinLimit,
-  ObjectReadLimitExceededError,
-  snapshotObjectReadExecutionLimits,
-} from "@sixb/core/storage"
+import { ObjectReadLimitExceededError, snapshotObjectReadExecutionLimits } from "@sixb/core/storage"
 import type { SQLClient, SqlParameter } from "../pg-client"
 import { type PgStoreClient, runPgRepeatableReadTransaction } from "../transactions"
 import { compilePgSelectedObjectReadSource, type PgSelectedObjectReadSource } from "./read-scope"
+import { PgObjectReader } from "./reader"
+import { createPgScopedReader } from "./scoped-reader"
 
-import { assertLinkQueryLimit, PgObjectReader } from "./reader"
-
+/** Keep the traversal probe and every terminal statement on one repeatable-read snapshot. */
 export function createPgSelectedReader(
   sql: PgStoreClient,
   params: {
@@ -33,74 +28,19 @@ export function createPgSelectedReader(
     params.scope,
     limits.maxTraversalFacts
   )
-  const assertProject = (actualProjectId: string): void =>
-    assertObjectReaderProject(projectId, actualProjectId)
-  const read = <T>(run: (reader: PgObjectReader) => Promise<T>): Promise<T> =>
-    runPgRepeatableReadTransaction(sql, async (tx) => {
-      await assertTraversalBudget(tx, source, limits.maxTraversalFacts)
-      const value = await run(new PgObjectReader(tx, source))
-      assertObjectReadOutputWithinLimit(value, limits)
-      return value
-    })
-  const readMap = <TKey, TValue>(
-    run: (reader: PgObjectReader) => Promise<Map<TKey, TValue>>
-  ): Promise<Map<TKey, TValue>> =>
-    runPgRepeatableReadTransaction(sql, async (tx) => {
-      await assertTraversalBudget(tx, source, limits.maxTraversalFacts)
-      const value = await run(new PgObjectReader(tx, source))
-      assertObjectReadOutputWithinLimit([...value.entries()], limits)
-      return value
-    })
 
-  return Object.freeze({
-    queryCapabilities,
-    queryObjects: async (input) => {
-      assertProject(input.projectId)
-      return read((reader) => reader.queryObjects(input))
-    },
-    countObjects: async (input) => {
-      assertProject(input.projectId)
-      return read((reader) => reader.countObjects(input))
-    },
-    existsObjects: async (input) => {
-      assertProject(input.projectId)
-      return read((reader) => reader.existsObjects(input))
-    },
-    facetObjects: async (input) => {
-      assertProject(input.projectId)
-      assertObjectReadFacetCount(input.facets.length)
-      return read((reader) => reader.facetObjects(input))
-    },
-    getByPrimaryId: async (input) => {
-      assertProject(input.projectId)
-      return read((reader) => reader.getByPrimaryId(input))
-    },
-    selectsObjectProperties: async (input) => {
-      assertProject(input.projectId)
-      return read((reader) => reader.selectsObjectProperties(input))
-    },
-    listLinks: async (input) => {
-      assertProject(input.projectId)
-      return read((reader) => reader.listLinks(input))
-    },
-    getByPrimaryIdBatch: async (input) => {
-      assertProject(input.projectId)
-      return readMap((reader) => reader.getByPrimaryIdBatch(input))
-    },
-    listLinksBatch: async (input) => {
-      assertProject(input.projectId)
-      return readMap((reader) => reader.listLinksBatch(input))
-    },
-    queryLinks: async (input) => {
-      assertProject(input.projectId)
-      assertLinkQueryLimit(input.limit)
-      return read((reader) => reader.queryLinks(input))
-    },
-    list: async (input) => {
-      assertProject(input.projectId)
-      return read((reader) => reader.list(input))
-    },
-  } satisfies ObjectReadStorage)
+  return createPgScopedReader(
+    {
+      projectId,
+      read: (run) =>
+        runPgRepeatableReadTransaction(sql, async (tx) => {
+          await assertTraversalBudget(tx, source, limits.maxTraversalFacts)
+          return run(new PgObjectReader(tx, source))
+        }),
+      limits,
+    },
+    queryCapabilities
+  )
 }
 
 async function assertTraversalBudget(

package/src/objects/storage.ts

@@ -1,5 +1,6 @@
 import { hasVectorProfile } from "@sixb/core/internal/query"
 import type {
+  CompiledConditionalObjectReadScope,
   CompiledSelectedObjectReadScope,
   CountObjectsInput,
   CountObjectsResult,
@@ -28,6 +29,7 @@ import {
   runPgRepeatableReadTransaction,
 } from "../transactions"
 import { PgAggregateAdmission } from "./aggregate-admission"
+import { createPgConditionalReader } from "./conditional-reader"
 import { DEFAULT_OBJECT_QUERY_SOURCE } from "./query-compiler"
 import { assertLinkQueryLimit, PgObjectReader } from "./reader"
 import {
@@ -71,6 +73,7 @@ const PG_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
     in: true,
     exists: true,
     contains: true,
+    constant: true,
   },
   sortKinds: {
     property: true,
@@ -110,17 +113,17 @@ const PG_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
 
 /** PostgreSQL object provider; interactive reads share one source-aware reader. */
 export class PgObjectStorage implements ObjectStorage {
+  private readonly aggregateAdmission?: PgAggregateAdmission
   private readonly reader: PgObjectReader
 
   constructor(
     private readonly sql: PgStoreClient,
     maxConcurrentAggregates = 1
   ) {
-    this.reader = new PgObjectReader(
-      sql,
-      DEFAULT_OBJECT_QUERY_SOURCE,
-      canStartPgTransaction(sql) ? new PgAggregateAdmission(maxConcurrentAggregates) : undefined
-    )
+    this.aggregateAdmission = canStartPgTransaction(sql)
+      ? new PgAggregateAdmission(maxConcurrentAggregates)
+      : undefined
+    this.reader = new PgObjectReader(sql, DEFAULT_OBJECT_QUERY_SOURCE, this.aggregateAdmission)
   }
 
   queryCapabilities(): ObjectQueryCapabilities {
@@ -135,6 +138,16 @@ export class PgObjectStorage implements ObjectStorage {
     return createPgSelectedReader(this.sql, params, () => this.queryCapabilities())
   }
 
+  createConditionalReadScope(params: {
+    projectId: string
+    scope: CompiledConditionalObjectReadScope
+  }): ObjectReadStorage {
+    return createPgConditionalReader(this.sql, params, {
+      aggregateAdmission: this.aggregateAdmission,
+      queryCapabilities: () => this.queryCapabilities(),
+    })
+  }
+
   async queryObjects(params: QueryObjectsInput): Promise<QueryObjectsResult> {
     if (hasVectorProfile(params.query))
       return runPgRepeatableReadTransaction(this.sql, (tx) =>
@@ -175,6 +188,12 @@ export class PgObjectStorage implements ObjectStorage {
     return this.reader.getByPrimaryId(params)
   }
 
+  async getByUniqueValue(
+    params: Parameters<ObjectReadStorage["getByUniqueValue"]>[0]
+  ): Promise<ObjectRow | null> {
+    return this.reader.getByUniqueValue(params)
+  }
+
   async selectsObjectProperties(
     params: Parameters<ObjectReadStorage["selectsObjectProperties"]>[0]
   ): Promise<readonly boolean[]> {

package/src/objects/text-count-index.ts

@@ -401,6 +401,7 @@ function stringPredicate(
       ? `(${items.join(predicate.op === "and" ? " AND " : " OR ")})`
       : undefined
   }
+  if (predicate.op === "constant") return predicate.value ? "true" : "false"
   if ((predicate.op !== "eq" && predicate.op !== "in") || predicate.scalarKind === "decimal")
     return undefined
   const column = columns.get(predicate.propertyId)

package/src/ontology-storage/index.ts

@@ -7,6 +7,7 @@ import { PgOntologyOutboxStorage } from "./outbox"
 import { PgOntologyReplacementPlanStorage } from "./replacement-plans"
 import type { PgRootOperation } from "./shared"
 import { PgOntologySourceStorage } from "./sources"
+import { PgOntologyUniqueValueStorage } from "./unique-values"
 import { PgVectorIndexingStorage } from "./vector-indexing"
 import { PgOntologyVectorStorage } from "./vectors"
 
@@ -18,6 +19,7 @@ export class PgOntologyStorage implements OntologyStorage {
   readonly outbox: PgOntologyOutboxStorage
   readonly vectorIndexing: PgVectorIndexingStorage
   readonly vectors: PgOntologyVectorStorage
+  readonly uniqueValues: PgOntologyUniqueValueStorage
 
   constructor(input: {
     readonly sql: SQLClient
@@ -32,6 +34,7 @@ export class PgOntologyStorage implements OntologyStorage {
     )
     this.replacementPlans = new PgOntologyReplacementPlanStorage(input.sql)
     this.outbox = new PgOntologyOutboxStorage(input.runRootOperation)
+    this.uniqueValues = new PgOntologyUniqueValueStorage(input.runRootOperation)
     this.vectorIndexing = new PgVectorIndexingStorage(
       input.sql,
       input.runRootOperation,

package/src/ontology-storage/materialization-session.ts

@@ -35,6 +35,9 @@ export class PgMaterializationSessionState extends ProviderMaterializationSessio
   stagedWorkCount = 0
   changedObjects = 0
   changedLinks = 0
+  changedUniqueValues = 0
+  /** What unique value items this session staged, known without reading its work back. */
+  readonly uniqueWork = { releases: false, claims: false }
   workAnalyzed = false
   /** Work staged on the session lives in a temp table; a plan-bound session reads its plan's. */
   readonly workTable: string
@@ -156,6 +159,11 @@ export class PgMaterializationSessions {
       throw error
     }
     session.stagedWorkCount += records.length
+    for (const { record } of records) {
+      if (record.kind !== "plan") continue
+      if (record.item.kind === "unique-value-release") session.uniqueWork.releases = true
+      if (record.item.kind === "unique-value-claim") session.uniqueWork.claims = true
+    }
   }
 
   /**

package/src/ontology-storage/materialization-writer.ts

@@ -16,6 +16,7 @@ import {
 import type { MaterializationPlanWorkItem } from "@sixb/core/storage"
 import type postgres from "postgres"
 import type { SQL, SQLClient } from "../pg-client"
+import { claimPgUniqueValues, heldValue } from "./unique-values"
 
 export interface PgStagedPlanTarget {
   /** The work table and the id its rows carry: a session's temp table, or a durable plan. */
@@ -24,11 +25,19 @@ export interface PgStagedPlanTarget {
   readonly projectId: string
   readonly commitId: string
   readonly committedAt: string
+  /** Whether unique value items were staged; probed in the work table when unknown. */
+  readonly uniqueWork?: PgUniqueWork
+}
+
+export interface PgUniqueWork {
+  readonly releases: boolean
+  readonly claims: boolean
 }
 
 export interface PgAppliedPlan {
   readonly objectWrites: number
   readonly linkWrites: number
+  readonly uniqueValueWrites: number
   readonly eventCount: number
 }
 
@@ -55,13 +64,32 @@ export class PgMaterializationWriter {
     await this.applyObjectOverrides(target)
     await this.applyLinkOverrides(target)
     await this.applyPoints(target)
+    const uniqueWork = target.uniqueWork ?? (await this.stagedUniqueWork(target))
+    if (uniqueWork.releases) await this.releaseUniqueValues(target)
     let linkWrites = await this.deleteLinks(target)
     let objectWrites = await this.deleteObjects(target)
     objectWrites += await this.upsertObjects(target)
     linkWrites += await this.upsertLinks(target)
+    const uniqueValueWrites = uniqueWork.claims ? await this.claimUniqueValues(target) : 0
     const eventCount = await this.writeOutbox(target)
     await this.writeTouches(target)
-    return { objectWrites, linkWrites, eventCount }
+    return { objectWrites, linkWrites, uniqueValueWrites, eventCount }
+  }
+
+  /** Two probes of the work table's lane index: a commit with no unique values runs nothing else. */
+  private async stagedUniqueWork(target: PgStagedPlanTarget): Promise<PgUniqueWork> {
+    const probe = (kind: "unique-value-release" | "unique-value-claim") => this.sql`
+      EXISTS (
+        SELECT 1 FROM ${this.sql(target.workTable)}
+        WHERE work_id = ${target.workId} AND lane = 'apply'
+          AND major_order = ${materializationApplyPhase(kind)}
+          AND minor_order = ${materializationPlanKindRank(kind)}
+      )
+    `
+    const [row] = await this.sql<{ readonly releases: boolean; readonly claims: boolean }[]>`
+      SELECT ${probe("unique-value-release")} AS releases, ${probe("unique-value-claim")} AS claims
+    `
+    return { releases: row?.releases ?? false, claims: row?.claims ?? false }
   }
 
   /** The staged values of one plan item kind, as `value`. */
@@ -330,6 +358,51 @@ export class PgMaterializationWriter {
     )
   }
 
+  /** The staged unique values of one item kind, as the columns their table keys them by. */
+  private stagedUniqueValues(
+    target: PgStagedPlanTarget,
+    kind: "unique-value-release" | "unique-value-claim"
+  ) {
+    return this.sql`
+      SELECT value->>'objectTypeId' AS object_type_id, value->>'propertyId' AS property_id,
+        value->>'valueKey' AS value_key, value->>'primaryId' AS primary_id,
+        value->>'keyKind' AS key_kind, (value->>'keyVersion')::integer AS key_version
+      FROM (${this.staged(target, kind)}) AS staged
+    `
+  }
+
+  /** A value whose row already names another object, or none, is simply no longer this one's. */
+  private async releaseUniqueValues(target: PgStagedPlanTarget): Promise<void> {
+    await this.sql`
+      WITH candidates AS MATERIALIZED (
+        ${this.stagedUniqueValues(target, "unique-value-release")}
+      ), released AS (
+        SELECT candidates.object_type_id, candidates.property_id, held.value_hash
+        FROM candidates
+        CROSS JOIN LATERAL (
+          ${heldValue(this.sql, target.projectId)}
+            AND stored.primary_id = candidates.primary_id
+          LIMIT 1
+        ) AS held
+      )
+      DELETE FROM object_unique_values AS stored USING released
+      WHERE stored.project_id = ${target.projectId}
+        AND stored.object_type_id = released.object_type_id
+        AND stored.property_id = released.property_id
+        AND stored.value_hash = released.value_hash
+    `
+  }
+
+  /** Runs once every release of the commit has applied, so values can move between objects. */
+  private async claimUniqueValues(target: PgStagedPlanTarget): Promise<number> {
+    return claimPgUniqueValues(
+      this.sql,
+      target.projectId,
+      this.stagedUniqueValues(target, "unique-value-claim"),
+      { requirePrepared: true }
+    )
+  }
+
   private async deleteLinks(target: PgStagedPlanTarget): Promise<number> {
     const candidates = this.staged(target, "link-delete")
     return this.write(
@@ -586,6 +659,8 @@ export class PgMaterializationWriter {
           END AS ref
       ) AS entity
       WHERE work.work_id = ${target.workId} AND work.lane = 'apply'
+        -- Unique values are no plan input: a stale claim fails when it applies.
+        AND entity.kind IS NOT NULL
     `
   }
 

package/src/ontology-storage/materializations.ts

@@ -63,6 +63,7 @@ import {
   toIsoString,
 } from "./shared"
 import { activateSourceRoots } from "./source-roots"
+import { pgUniqueHolders } from "./unique-values"
 
 interface TelemetrySummaryRow {
   readonly classified_points: number | string
@@ -203,6 +204,17 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
           yield { objects: [], links: [], linkScopes: [], points: stored }
         }
       }
+      const uniqueValues = request.uniqueValues ?? []
+      for (let offset = 0; offset < uniqueValues.length; offset += input.pageRows) {
+        const uniqueHolders = await pgUniqueHolders(
+          this.sql,
+          session.header.commit.projectId,
+          uniqueValues.slice(offset, offset + input.pageRows)
+        )
+        if (uniqueHolders.length > 0) {
+          yield { objects: [], links: [], linkScopes: [], points: [], uniqueHolders }
+        }
+      }
     }
   }
 
@@ -238,9 +250,11 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
       projectId: commit.projectId,
       commitId: commit.id,
       committedAt: commit.committedAt,
+      ...(session.plan ? {} : { uniqueWork: session.uniqueWork }),
     })
     session.changedObjects = applied.objectWrites
     session.changedLinks = applied.linkWrites
+    session.changedUniqueValues = applied.uniqueValueWrites
     session.appliedEventCount = applied.eventCount
     return { eventCount: applied.eventCount }
   }
@@ -256,6 +270,11 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
     // With empty/stale statistics, a link lookup can scan every edge sharing its target.
     await refreshMaterializedStatistics(this.sql, "objects", session.changedObjects)
     await refreshMaterializedStatistics(this.sql, "links", session.changedLinks)
+    await refreshMaterializedStatistics(
+      this.sql,
+      "object_unique_values",
+      session.changedUniqueValues
+    )
     const record = await this.insertCommit(session.header, input)
     await this.sessions.release(session)
     return { commit: record }
@@ -743,7 +762,7 @@ function databaseCount(value: number | string | undefined): number {
  * count can still include a previous transaction's manually analyzed writes at this point. */
 async function refreshMaterializedStatistics(
   sql: SQLClient,
-  table: "objects" | "links",
+  table: "objects" | "links" | "object_unique_values",
   changed: number
 ): Promise<void> {
   if (changed < 1_000) return