@sixb/core 0.1.16 → 0.1.18

package/src/objects/property-clearance.ts

@@ -0,0 +1,209 @@
+import { AuthorizationError } from "../authorization/errors"
+import type { StoredDomainEvent } from "../events/types"
+import type { OntologyDefinitionCatalog } from "../ontology/registry"
+import { missingClearanceRedactions } from "../security/redactions"
+import type { ExpandedLinkValue, ExpandedObjectRow, ObjectRow } from "../storage"
+import type {
+  ObjectQueryAdmissionState,
+  ObjectQueryPropertyUse,
+  ObjectQuerySemanticAdmission,
+} from "./query/validate"
+
+/**
+ * The marked properties one reader cannot read, resolved once from its clearances.
+ *
+ * A property is readable when the reader clears every one of its markings. Unreadable properties
+ * are omitted from rows and cannot be used to select, order, or aggregate objects.
+ */
+export interface PropertyClearance {
+  /** Marked properties of this object type the reader cannot read. */
+  hiddenPropertyIds(objectTypeId: string): ReadonlySet<string>
+  /** First marking on the property that the reader is not cleared for. */
+  missingMarking(objectTypeId: string, propertyId: string): string | undefined
+}
+
+const NO_PROPERTIES: ReadonlySet<string> = new Set()
+
+// Readers are built per execution, but few distinct clearance sets exist per ontology.
+const clearancesByOntology = new WeakMap<
+  OntologyDefinitionCatalog,
+  Map<string, PropertyClearance | undefined>
+>()
+
+/**
+ * Resolve what a reader with these clearances cannot read. Returns `undefined` when it can read
+ * every property, so callers skip redaction entirely.
+ */
+export function resolvePropertyClearance(
+  ontology: OntologyDefinitionCatalog,
+  clearances: ReadonlySet<string>
+): PropertyClearance | undefined {
+  let resolved = clearancesByOntology.get(ontology)
+  if (!resolved) {
+    resolved = new Map()
+    clearancesByOntology.set(ontology, resolved)
+  }
+  const key = JSON.stringify([...clearances].sort())
+  if (!resolved.has(key)) resolved.set(key, createPropertyClearance(ontology, clearances))
+  return resolved.get(key)
+}
+
+function createPropertyClearance(
+  ontology: OntologyDefinitionCatalog,
+  clearances: ReadonlySet<string>
+): PropertyClearance | undefined {
+  const missingByType = new Map<string, ReadonlyMap<string, string>>()
+  for (const objectType of ontology.listObjectTypes()) {
+    const missing = new Map<string, string>()
+    for (const property of objectType.properties) {
+      const marking = property.markings?.find((markingId) => !clearances.has(markingId))
+      if (marking !== undefined) missing.set(property.id, marking)
+    }
+    if (missing.size > 0) missingByType.set(objectType.id, missing)
+  }
+  if (missingByType.size === 0) return undefined
+
+  const hiddenByType = new Map(
+    [...missingByType].map(([objectTypeId, missing]) => [objectTypeId, new Set(missing.keys())])
+  )
+  return Object.freeze({
+    hiddenPropertyIds: (objectTypeId: string) => hiddenByType.get(objectTypeId) ?? NO_PROPERTIES,
+    missingMarking: (objectTypeId: string, propertyId: string) =>
+      missingByType.get(objectTypeId)?.get(propertyId),
+  })
+}
+
+/**
+ * Omit unreadable properties from a row and the objects expanded under it.
+ *
+ * Every hidden property of the type is listed in `redactions`, whether or not the row holds a
+ * value: listing only present values would reveal which hidden properties are empty.
+ */
+export function redactObjectRow<TRow extends ObjectRow>(
+  row: TRow,
+  clearance: PropertyClearance
+): TRow {
+  const hidden = clearance.hiddenPropertyIds(row.objectTypeId)
+  const links = row.links
+  if (hidden.size === 0 && links === undefined) return row
+
+  const redacted: TRow = { ...row }
+  if (hidden.size > 0) {
+    redacted.properties = withoutKeys(row.properties, hidden)
+    redacted.redactions = missingClearanceRedactions(hidden)
+  }
+  if (links !== undefined) {
+    redacted.links = Object.fromEntries(
+      Object.entries(links).map(([linkId, value]) => [linkId, redactLinkValue(value, clearance)])
+    )
+  }
+  return redacted
+}
+
+function redactLinkValue(
+  value: ExpandedLinkValue,
+  clearance: PropertyClearance
+): ExpandedLinkValue {
+  if (value === null) return null
+  if (Array.isArray(value)) return value.map((row) => redactObjectRow(row, clearance))
+  // Array.isArray does not narrow a readonly[] away, so assert the single-object case.
+  return redactObjectRow(value as ExpandedObjectRow, clearance)
+}
+
+/**
+ * Omit unreadable properties from an object event, under the same contract as rows. Other events
+ * carry no markable value: link properties and telemetry properties cannot be marked.
+ */
+export function redactDomainEvent(
+  event: StoredDomainEvent,
+  clearance: PropertyClearance
+): StoredDomainEvent {
+  if (event.topic !== "objects") return event
+  const hidden = clearance.hiddenPropertyIds(event.payload.objectTypeId)
+  if (hidden.size === 0) return event
+
+  const payload = {
+    ...event.payload,
+    propertyChanges: withoutKeys(event.payload.propertyChanges, hidden),
+    redactions: missingClearanceRedactions(hidden),
+  }
+  if (event.type === "object.deleted") return { ...event, payload }
+  return {
+    ...event,
+    payload: { ...payload, properties: withoutKeys(event.payload.properties, hidden) },
+  }
+}
+
+function withoutKeys<TValue>(
+  record: Readonly<Record<string, TValue>>,
+  hidden: ReadonlySet<string>
+): Record<string, TValue> {
+  const kept: Record<string, TValue> = {}
+  for (const [key, value] of Object.entries(record)) {
+    if (!hidden.has(key)) kept[key] = value
+  }
+  return kept
+}
+
+/**
+ * Reject every query use of a property the reader cannot read, except projection.
+ *
+ * Filtering, ordering, searching, or faceting by a hidden value would reveal it through which
+ * objects match and in what order. Projection only shapes the output, which is redacted anyway.
+ */
+export function createClearanceQueryAdmission(
+  clearance: PropertyClearance
+): ObjectQuerySemanticAdmission {
+  // Each state stands for the object types a query has reached so far. A property without an
+  // explicit type is checked against all of them.
+  const reachedTypes = new WeakMap<ObjectQueryAdmissionState, readonly string[]>()
+  const state = (objectTypeIds: readonly string[]): ObjectQueryAdmissionState => {
+    const created = Object.freeze({})
+    reachedTypes.set(created, Object.freeze([...objectTypeIds]))
+    return created
+  }
+  const reached = (input: ObjectQueryAdmissionState): readonly string[] => {
+    const objectTypeIds = reachedTypes.get(input)
+    if (!objectTypeIds) {
+      throw new Error("[Sixb] Clearance query admission received a state it did not create.")
+    }
+    return objectTypeIds
+  }
+  const empty = state([])
+
+  const admission: ObjectQuerySemanticAdmission = {
+    empty: () => empty,
+    source: (input) => ({ state: state(input.result.objectTypeIds) }),
+    edge: (input) => ({ state: state(input.result.objectTypeIds) }),
+    // A difference only returns objects of its first input.
+    set: (input) =>
+      input.op === "subtract"
+        ? (input.states[0] ?? empty)
+        : state([...new Set(input.states.flatMap(reached))]),
+    property: (input) => {
+      if (input.use === "project") return undefined
+      const objectTypeIds =
+        input.objectTypeId === undefined ? reached(input.state) : [input.objectTypeId]
+      for (const objectTypeId of objectTypeIds) {
+        const marking = clearance.missingMarking(objectTypeId, input.propertyId)
+        if (marking === undefined) continue
+        return new AuthorizationError(
+          `clearance:marking:${marking}`,
+          `[Sixb] Cannot ${PROPERTY_USE_VERBS[input.use]} '${objectTypeId}.${input.propertyId}' at '${input.path}': it requires clearance for marking '${marking}'.`
+        )
+      }
+      return undefined
+    },
+  }
+  return Object.freeze(admission)
+}
+
+const PROPERTY_USE_VERBS: Readonly<Record<ObjectQueryPropertyUse, string>> = {
+  filter: "filter by",
+  text: "search",
+  vector: "run a vector search over",
+  sort: "sort by",
+  project: "project",
+  "expand.orderBy": "order expanded objects by",
+  facet: "facet by",
+}

package/src/objects/query/executor.ts

@@ -1,6 +1,6 @@
 import { type AuthorizationContext, assertAuthorized } from "../../authorization"
 import type { RuntimeAuthorization } from "../../execution/types"
-import type { EmbeddingModelCatalog } from "../../models/catalog"
+import type { EmbeddingModelCatalog, RerankingModelCatalog } from "../../models/catalog"
 import type { OntologyRegistry } from "../../ontology"
 import type {
   CountObjectsResult,
@@ -34,12 +34,18 @@ import type {
 } from "./ir"
 import { normalizeObjectQuery } from "./normalize"
 import { type ObjectQueryPlan, type ObjectQueryPlanningOptions, planObjectQuery } from "./planner"
+import { usePrimaryIdLookups } from "./primary-id"
+import { rerankObjectCandidates } from "./reranking-executor"
+import { findQueryReranking } from "./reranking-query"
 import { compareQueryScalarValues, queryScalarValuesEqual } from "./scalar-values"
 import {
+  type AdmittedObjectQuery,
+  type ObjectQuerySemanticAdmission,
   type ObjectQueryValidationIssue,
   resolveObjectQueryResultShape,
   type ValidatedObjectQuery,
   validateObjectQuery,
+  validateObjectQueryWithAdmission,
 } from "./validate"
 
 export interface QueryExecutorOptions
@@ -54,6 +60,7 @@ export interface QueryExecutorOptions
   > {
   ontology: OntologyRegistry
   embeddingModels?: EmbeddingModelCatalog
+  rerankingModels?: RerankingModelCatalog
   storage: ObjectReadStorage
   maxLimit?: number
   maxPageSize?: number
@@ -69,6 +76,8 @@ export interface QueryExecutorOptions
   authorization?: AuthorizationContext
   /** Registered execution capability for queries reached through a bound Sixb SDK. */
   runtimeAuthorization?: RuntimeAuthorization
+  /** Reader-specific admission applied after grants, such as marking clearances. */
+  admission?: ObjectQuerySemanticAdmission
 }
 
 export interface ExecuteObjectQueryInput {
@@ -134,15 +143,7 @@ export async function executeObjectQuery(
   input: ExecuteObjectQueryInput,
   options: QueryExecutorOptions
 ): Promise<ExecuteObjectQueryResult> {
-  const normalized = normalizeObjectQuery(input.query)
-  const validated = validateObjectQuery(normalized, {
-    ontology: options.ontology,
-    maxLimit: options.maxLimit,
-    maxPageSize: options.maxPageSize,
-    maxRefs: options.maxRefs,
-    normalize: false,
-  })
-  assertQueryViewable(input.projectId, validated, options)
+  const validated = validateExecutableQuery(input, options)
   const capabilities = options.storage.queryCapabilities()
   const hasQueryObjects = typeof options.storage.queryObjects === "function"
   const hasCountObjects = typeof options.storage.countObjects === "function"
@@ -178,6 +179,37 @@ export async function executeObjectQuery(
     throw new ObjectQueryPlanningError(plan.issues)
   }
 
+  if (plan.mode === "pipeline") {
+    const reranking = findQueryReranking(validated.query)
+    if (!reranking || reranking.input.kind !== "vector") {
+      throw new ObjectQueryExecutionError(
+        "invalid_rerank_input",
+        "Reranking requires vector candidates."
+      )
+    }
+    // Resolve the allowlisted server binding before paying for query embedding.
+    const model = options.rerankingModels?.getByRef(reranking.model)?.model
+    if (!model) {
+      throw new ObjectQueryExecutionError(
+        "reranking_model_unavailable",
+        `Reranking model '${reranking.model.provider}/${reranking.model.modelId}' is not configured in models.reranking.`
+      )
+    }
+    const candidates = await executeObjectQuery(
+      { ...input, query: reranking.input, includeTotal: false },
+      options
+    )
+    const result = await rerankObjectCandidates({
+      query: validated.query,
+      vector: reranking.input,
+      candidates: candidates.objects,
+      model,
+      signal: input.signal,
+      includeTotal: input.includeTotal,
+    })
+    return { ...result, plan }
+  }
+
   if (plan.mode === "pushdown") {
     if (!options.storage.queryObjects) {
       throw new ObjectQueryPlanningError(plan.providerIssues)
@@ -207,15 +239,7 @@ export async function countObjects(
   input: ExecuteObjectCountInput,
   options: QueryExecutorOptions
 ): Promise<ExecuteObjectCountResult> {
-  const normalized = normalizeObjectQuery(input.query)
-  const validated = validateObjectQuery(normalized, {
-    ontology: options.ontology,
-    maxLimit: options.maxLimit,
-    maxPageSize: options.maxPageSize,
-    maxRefs: options.maxRefs,
-    normalize: false,
-  })
-  assertQueryViewable(input.projectId, validated, options)
+  const validated = validateExecutableQuery(input, options)
   const capabilities = options.storage.queryCapabilities()
   const hasQueryObjects = typeof options.storage.queryObjects === "function"
   const hasCountObjects = typeof options.storage.countObjects === "function"
@@ -268,15 +292,7 @@ export async function existsObjects(
   input: ExecuteObjectExistsInput,
   options: QueryExecutorOptions
 ): Promise<ExecuteObjectExistsResult> {
-  const normalized = normalizeObjectQuery(input.query)
-  const validated = validateObjectQuery(normalized, {
-    ontology: options.ontology,
-    maxLimit: options.maxLimit,
-    maxPageSize: options.maxPageSize,
-    maxRefs: options.maxRefs,
-    normalize: false,
-  })
-  assertQueryViewable(input.projectId, validated, options)
+  const validated = validateExecutableQuery(input, options)
   const capabilities = options.storage.queryCapabilities()
   const hasQueryObjects = typeof options.storage.queryObjects === "function"
   const hasCountObjects = typeof options.storage.countObjects === "function"
@@ -329,16 +345,17 @@ export async function facetObjects(
   input: ExecuteObjectFacetsInput,
   options: QueryExecutorOptions
 ): Promise<ExecuteObjectFacetsResult> {
-  const normalized = normalizeObjectQuery(input.query)
-  const validated = validateObjectQuery(normalized, {
-    ontology: options.ontology,
-    maxLimit: options.maxLimit,
-    maxPageSize: options.maxPageSize,
-    maxRefs: options.maxRefs,
-    normalize: false,
-  })
-  assertQueryViewable(input.projectId, validated, options)
+  const validated = validateExecutableQuery(input, options)
   const facets = validateObjectFacetRequests(input.facets, validated.result.objectTypeIds, options)
+  facets.forEach((facet, index) => {
+    const denial = options.admission?.property({
+      state: validated.admissionState,
+      propertyId: facet.propertyId,
+      use: "facet",
+      path: `$.facets[${index}].propertyId`,
+    })
+    if (denial) throw denial
+  })
   const aggregateQuery = stripOuterRowShape(validated.query)
   const capabilities = options.storage.queryCapabilities()
   const hasQueryObjects = typeof options.storage.queryObjects === "function"
@@ -386,6 +403,7 @@ export async function facetObjects(
   const maxRows = options.maxFallbackRows ?? DEFAULT_MAX_FALLBACK_ROWS
   const evaluation = await evaluateFallbackQuery(input.projectId, aggregateQuery, options, maxRows)
   return {
+    total: evaluation.entries.length,
     facets: buildFacetResults(
       evaluation.entries.map((entry) => entry.row),
       facets
@@ -394,6 +412,31 @@ export async function facetObjects(
   }
 }
 
+/**
+ * Validate, authorize the touched types, then admit the query for the reader.
+ *
+ * Grants are checked first so a denial never describes properties of a type the caller cannot
+ * view. The admitted query is the one executed: an admission may narrow it, as markings narrow
+ * default text fields.
+ */
+function validateExecutableQuery(
+  input: { readonly projectId: string; readonly query: ObjectQuery },
+  options: QueryExecutorOptions
+): AdmittedObjectQuery {
+  const validationOptions = {
+    ontology: options.ontology,
+    maxLimit: options.maxLimit,
+    maxPageSize: options.maxPageSize,
+    maxRefs: options.maxRefs,
+    normalize: false,
+  }
+  const normalized = normalizeObjectQuery(input.query)
+  const validated = validateObjectQuery(normalized, validationOptions)
+  assertQueryViewable(input.projectId, validated, options)
+  if (!options.admission) return { ...validated, admissionState: {} }
+  return validateObjectQueryWithAdmission(normalized, validationOptions, options.admission)
+}
+
 // Authorization happens at planning time: a scoped query must hold a view
 // grant for every object type it touches, before any storage call runs.
 function assertQueryViewable(
@@ -432,7 +475,7 @@ function expandPushdownQuery(
   // disagree) cleanly stays on the fallback.
   const ctx: ExpansionResolutionContext = { ontology, maxExpansionFanout }
   const annotated = resolveExpansions(query, ctx)
-  const expanded = expandIncludeSubtypes(annotated, ontology)
+  const expanded = usePrimaryIdLookups(expandIncludeSubtypes(annotated, ontology), ontology)
   if (expanded === annotated) return annotated
 
   const plan = planObjectQuery(expanded, { ...planning, capabilities, allowFallback: false })
@@ -465,6 +508,7 @@ function resolveExpansions(query: ObjectQuery, ctx: ExpansionResolutionContext):
     case "filter":
     case "text":
     case "vector":
+    case "rerank":
     case "traverse":
     case "sort":
     case "limit":
@@ -581,6 +625,7 @@ function expandIncludeSubtypes(query: ObjectQuery, ontology: OntologyRegistry):
     case "filter":
     case "text":
     case "vector":
+    case "rerank":
     case "traverse":
     case "sort":
     case "limit":
@@ -688,6 +733,7 @@ async function evaluateFallbackQuery(
     }
     case "text":
     case "vector":
+    case "rerank":
     case "traverse":
     case "set":
       throw new ObjectQueryExecutionError(
@@ -1186,6 +1232,7 @@ function stripOuterRowShape(query: ObjectQuery): ObjectQuery {
     case "page":
     case "project":
     case "sort":
+    case "rerank":
     // `expand` is output-shaping: it attaches links but does not change which
     // objects match, so aggregates (count/exists/facets) ignore it.
     case "expand":

package/src/objects/query/explain.ts

@@ -145,6 +145,14 @@ function buildExplainNode(query: ObjectQuery, path: string): ObjectQueryExplainN
         },
         children: [buildExplainNode(query.input, `${path}.input`)],
       }
+    case "rerank":
+      return {
+        path,
+        kind: query.kind,
+        summary: `rerank ${query.model.provider}/${query.model.modelId}`,
+        details: { model: query.model },
+        children: [buildExplainNode(query.input, `${path}.input`)],
+      }
     case "set":
       return {
         path,

package/src/objects/query/index.ts

@@ -47,6 +47,7 @@ export type {
   ObjectQueryPredicateNot,
   ObjectQueryProject,
   ObjectQueryRefs,
+  ObjectQueryRerank,
   ObjectQueryResultShape,
   ObjectQuerySet,
   ObjectQuerySetOperation,

package/src/objects/query/ir.ts

@@ -5,6 +5,8 @@
  * storage backend. Providers can push down the subset they support, while the
  * core planner decides whether a bounded fallback is acceptable.
  */
+
+import type { ModelRef } from "../../models/catalog"
 import type { ObjectRef } from "../../ontology/refs"
 
 export type ObjectQueryDirection = "outgoing" | "incoming"
@@ -29,6 +31,7 @@ export type ObjectQuery =
   | ObjectQueryFilter
   | ObjectQueryText
   | ObjectQueryVector
+  | ObjectQueryRerank
   | ObjectQueryTraverse
   | ObjectQuerySet
   | ObjectQuerySort
@@ -102,6 +105,13 @@ export interface ObjectQueryTraverse {
   sourceObjectTypeId?: string
 }
 
+/** Explicit server-side reordering of a bounded vector candidate set. */
+export interface ObjectQueryRerank {
+  kind: "rerank"
+  input: ObjectQuery
+  model: ModelRef
+}
+
 export interface ObjectQuerySet {
   kind: "set"
   op: ObjectQuerySetOperation

package/src/objects/query/normalize.ts

@@ -46,6 +46,8 @@ function normalizeNode(query: ObjectQuery): ObjectQuery {
       }
     case "traverse":
       return { ...query, input: normalizeObjectQuery(query.input) }
+    case "rerank":
+      return { ...query, model: { ...query.model }, input: normalizeObjectQuery(query.input) }
     case "set":
       return normalizeSet(query)
     case "sort":

package/src/objects/query/planner.ts

@@ -1,8 +1,9 @@
 import type { ObjectQueryCapabilities, ObjectQueryScalarOperation } from "../../storage"
 import type { ObjectQueryPlanningIssue } from "./errors"
 import type { ObjectExpansion, ObjectQuery, ObjectQueryPredicate, ObjectQuerySortField } from "./ir"
+import { findQueryReranking } from "./reranking-query"
 
-export type ObjectQueryPlanMode = "pushdown" | "fallback" | "rejected"
+export type ObjectQueryPlanMode = "pushdown" | "fallback" | "pipeline" | "rejected"
 export type ObjectQueryProviderOperation =
   | "queryObjects"
   | "countObjects"
@@ -54,6 +55,15 @@ export function planObjectQuery(
   options: ObjectQueryPlanningOptions
 ): ObjectQueryPlan {
   const operation = options.operation ?? "queryObjects"
+  const reranking = findQueryReranking(query)
+  if (reranking) {
+    const candidatePlan = planObjectQuery(reranking.input, options)
+    return {
+      ...candidatePlan,
+      query,
+      mode: candidatePlan.mode === "rejected" ? "rejected" : "pipeline",
+    }
+  }
   // Providers get first refusal. Fallback is considered only after the provider
   // declares that pushdown is unavailable or incomplete for this query.
   const providerIssues = collectProviderIssues(query, options.capabilities, {
@@ -204,6 +214,7 @@ function collectNodeProviderIssues(
       return
     case "text":
     case "vector":
+    case "rerank":
     case "project":
       collectNodeProviderIssues(query.input, `${path}.input`, capabilities, issues)
       return

package/src/objects/query/primary-id.ts

@@ -0,0 +1,56 @@
+import type { OntologyDefinitionCatalog } from "../../ontology/registry"
+import type { ObjectQuery, ObjectQueryPredicate } from "./ir"
+
+/** Exact identity predicates can start from primary-key lookups. Keep the original filter:
+ * the optimization must not remove other conditions or expose a projected-out property.
+ * Only a direct start is rewritten; filters after bounds/projections keep their semantics. */
+export function usePrimaryIdLookups(
+  query: ObjectQuery,
+  ontology: OntologyDefinitionCatalog
+): ObjectQuery {
+  if (query.kind === "set")
+    return { ...query, inputs: query.inputs.map((input) => usePrimaryIdLookups(input, ontology)) }
+  if (!("input" in query)) return query
+  const input = usePrimaryIdLookups(query.input, ontology)
+  if (query.kind === "filter" && input.kind === "start" && !input.includeSubtypes) {
+    const propertyId = ontology.getPrimaryPropertyId(input.objectTypeId)
+    const ids = primaryIds(query.predicate, propertyId)
+    if (ids?.length && ids.length <= 1000)
+      return {
+        ...query,
+        input: {
+          kind: "refs",
+          refs: [...new Set(ids)]
+            .sort()
+            .map((primaryId) => ({ objectTypeId: input.objectTypeId, primaryId })),
+        },
+      }
+  }
+  return input === query.input ? query : { ...query, input }
+}
+
+function primaryIds(
+  predicate: ObjectQueryPredicate,
+  propertyId: string
+): readonly string[] | undefined {
+  if (predicate.op === "and") {
+    for (const item of predicate.items) {
+      const ids = primaryIds(item, propertyId)
+      if (ids) return ids
+    }
+    return undefined
+  }
+  if (
+    predicate.op === "eq" &&
+    predicate.propertyId === propertyId &&
+    typeof predicate.value === "string"
+  )
+    return [predicate.value]
+  if (
+    predicate.op === "in" &&
+    predicate.propertyId === propertyId &&
+    predicate.values.every((value) => typeof value === "string")
+  )
+    return predicate.values as readonly string[]
+  return undefined
+}