@push.rocks/smartdb 5.5.1 → 5.6.0

package/dist_rust/rustdb_linux_amd64.tsrust-build.json

@@ -1,12 +1,12 @@
 {
   "format": "tsrust.build-provenance.v2",
-  "binarySha256": "a4b1bb92556ac16600d9064d672acedebbada2ae471101aea54d3f81ca2aa14e",
+  "binarySha256": "55347a7396635075da1027b20e2bbccff00424f012d2001c087c73ea7e6c414d",
   "buildInfo": {
     "projectName": "@push.rocks/smartdb",
-    "projectVersion": "5.5.1",
-    "gitCommit": "6f369809b33f1a1e98ea51abf401e083dbd4818c",
+    "projectVersion": "5.6.0",
+    "gitCommit": "f0db2bc4f73ea31fd63f912dc61036c914946fce",
     "gitDirty": false,
-    "builtAt": "2026-09-02T16:42:20.006Z",
+    "builtAt": "2026-09-05T12:44:09.182Z",
     "tsrustVersion": "1.10.4",
     "binary": "rustdb",
     "target": "linux_amd64"

package/dist_rust/rustdb_linux_arm64.tsrust-build.json

@@ -1,12 +1,12 @@
 {
   "format": "tsrust.build-provenance.v2",
-  "binarySha256": "bb68d54102556aff95978fa1e70bc38318ebb5508fdae4d1a0c1aac9903476a0",
+  "binarySha256": "14467960f6674176bc91a7237a471459a415b8e699c3202541e7850e389627f9",
   "buildInfo": {
     "projectName": "@push.rocks/smartdb",
-    "projectVersion": "5.5.1",
-    "gitCommit": "6f369809b33f1a1e98ea51abf401e083dbd4818c",
+    "projectVersion": "5.6.0",
+    "gitCommit": "f0db2bc4f73ea31fd63f912dc61036c914946fce",
     "gitDirty": false,
-    "builtAt": "2026-09-02T16:42:58.586Z",
+    "builtAt": "2026-09-05T12:44:47.197Z",
     "tsrustVersion": "1.10.4",
     "binary": "rustdb",
     "target": "linux_arm64"

package/dist_rust/rustdb_macos_amd64.tsrust-build.json

@@ -1,12 +1,12 @@
 {
   "format": "tsrust.build-provenance.v2",
-  "binarySha256": "76c1670995d9a3d73aa27a2a071ac38483493683b0dbc0615c1570de96ed8dde",
+  "binarySha256": "255233b261d637b0ef65245ba123672b4fb89393127e92647bde47a9da4d07d7",
   "buildInfo": {
     "projectName": "@push.rocks/smartdb",
-    "projectVersion": "5.5.1",
-    "gitCommit": "6f369809b33f1a1e98ea51abf401e083dbd4818c",
+    "projectVersion": "5.6.0",
+    "gitCommit": "f0db2bc4f73ea31fd63f912dc61036c914946fce",
     "gitDirty": false,
-    "builtAt": "2026-09-02T16:44:04.443Z",
+    "builtAt": "2026-09-05T12:45:49.186Z",
     "tsrustVersion": "1.10.4",
     "binary": "rustdb",
     "target": "macos_amd64"

package/dist_rust/rustdb_macos_arm64.tsrust-build.json

@@ -1,12 +1,12 @@
 {
   "format": "tsrust.build-provenance.v2",
-  "binarySha256": "49cb67fc38b85d8d50c86bd9fae504809b2439a3c174e707e688f61eada5870b",
+  "binarySha256": "c90862d452ac29941a98e94649cdf255cf93fdc895a2c0be61aa1c82bf146bbe",
   "buildInfo": {
     "projectName": "@push.rocks/smartdb",
-    "projectVersion": "5.5.1",
-    "gitCommit": "6f369809b33f1a1e98ea51abf401e083dbd4818c",
+    "projectVersion": "5.6.0",
+    "gitCommit": "f0db2bc4f73ea31fd63f912dc61036c914946fce",
     "gitDirty": false,
-    "builtAt": "2026-09-02T16:44:48.396Z",
+    "builtAt": "2026-09-05T12:46:28.431Z",
     "tsrustVersion": "1.10.4",
     "binary": "rustdb",
     "target": "macos_arm64"

package/dist_ts/00_commitinfo_data.js

@@ -3,7 +3,7 @@
  */
 export const commitinfo = {
     name: '@push.rocks/smartdb',
-    version: '5.5.1',
+    version: '5.6.0',
     description: 'A MongoDB-compatible embedded database server with wire protocol support, backed by a high-performance Rust engine.'
 };
 //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiMDBfY29tbWl0aW5mb19kYXRhLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vdHMvMDBfY29tbWl0aW5mb19kYXRhLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBOztHQUVHO0FBQ0gsTUFBTSxDQUFDLE1BQU0sVUFBVSxHQUFHO0lBQ3hCLElBQUksRUFBRSxxQkFBcUI7SUFDM0IsT0FBTyxFQUFFLE9BQU87SUFDaEIsV0FBVyxFQUFFLHFIQUFxSDtDQUNuSSxDQUFBIn0=

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartdb",
-  "version": "5.5.1",
+  "version": "5.6.0",
   "private": false,
   "description": "A MongoDB-compatible embedded database server with wire protocol support, backed by a high-performance Rust engine.",
   "exports": {

package/readme.md

@@ -840,6 +840,13 @@ Mixed `localField`/`foreignField` plus `pipeline` lookups and other inner stages
 are rejected explicitly. Complete filter trees are validated before data is
 read, and query nesting plus aggregation work are bounded.
 
+A leading `$match` narrows the pipeline's source load instead of filtering
+after it: only matching documents are materialized, using an equality index when
+one serves the filter and a bounded scan otherwise. Filter shapes the query
+matcher cannot pre-compile fall back to the unnarrowed load, so results are
+unchanged. The materialization limit therefore applies to the matched set, which
+is what lets `countDocuments(filter)` run against collections larger than it.
+
 **Group accumulators:** `$sum`, `$avg`, `$min`, `$max`, `$first`, `$last`, `$push`, `$addToSet`, `$count`
 
 ### Indexes
@@ -862,6 +869,49 @@ validated no-op and index version `v: 2`. Unsupported options, including
 index in the request is created. TTL metadata is retained in the catalog;
 automatic TTL expiry is not implemented.
 
+`find()` and collection `aggregate()` accept `hint` as an index name (including
+`_id_`) or an exact ascending/descending key pattern. Key-pattern order matters;
+if several indexes share a key pattern, use a name. The selected active scalar
+index supplies every candidate, using safe full-key equality bounds or a full
+scan of that index. The complete predicate still runs, and explicit sorting,
+projection, skip, and limit retain their normal semantics. Index traversal uses
+bounded pages under the namespace lock, so concurrent writes wait until the
+hinted source read finishes. Unsorted results have no guaranteed BSON value order.
+
+```typescript
+await collection.createIndex({ organizationId: 1, userId: 1 }, { name: 'organization_user' });
+const page = await collection.find({ organizationId: 'org-1' }, {
+  hint: 'organization_user', sort: { userId: 1, _id: 1 }, skip: 20, limit: 10,
+}).toArray();
+const count = await collection.countDocuments({ organizationId: 'org-1' }, {
+  hint: 'organization_user',
+});
+```
+
+Unknown or quarantined indexes, malformed/ambiguous patterns, `$natural`, and
+hints on missing collections or views fail explicitly. Forced multikey traversal
+is unsupported and rejects; unhinted array queries keep their existing scan
+behavior. Sparse hints intentionally exclude documents absent from that index,
+including for empty filters and counts. A hint is never silently replaced by a
+collection scan or another index.
+
+Hinted transaction reads construct the selected index from the existing bounded
+transaction snapshot plus buffered writes; they never fetch candidate documents
+from live storage. Read-only aggregate source pipelines also use that transaction
+view. Foreign-collection stages (`$lookup`, `$unionWith`) and writing stages
+(`$out`, `$merge`), including stages inside `$facet`, reject within transactions.
+Their existing nontransaction behavior is unchanged.
+
+Finite count probes consisting of optional `$match`, optional `$skip`, positive
+integer `$limit`, and `$count` (or the driver's equivalent constant-one `$group`)
+stream candidates and stop when the limit is reached. They preserve hints,
+deadlines, and empty-result shapes without materializing the matched set. Stage
+dispatch, syntax, specifications, predicates, and streamed BSON processing consume
+the same aggregation work budget as ordinary pipelines. This permits bounded namespace inspection of collections larger
+than 10,000 documents or 32 MiB. Generic pipelines retain their existing
+materialization limits. Transactions still require the existing bounded base
+snapshot; finite counts do not bypass its 10,000-document / 32 MiB admission cap.
+
 ### Database & Admin
 
 ```typescript

package/readme.plan.md

@@ -1,336 +1,9 @@
-# SmartDB advancement plan
+# Query hints and finite counts
 
-Status: canonical direction, approved 2026-08-17. Reread before starting engine or
-managed-service work.
+Find and aggregate select the requested active scalar index by name or exact numeric key pattern. Candidate pages have count and byte bounds; full predicates, explicit sorting, projection, skip, limit, deadlines, and cursor limits remain authoritative. Unknown, malformed, ambiguous, natural-order, and multikey hints reject explicitly. Sparse hints retain sparse membership semantics.
 
-Baseline: `@push.rocks/smartdb` 5.5.0. SmartDB is a Rust database engine with a
-TypeScript lifecycle facade, a MongoDB wire-protocol command surface, file and
-memory storage, authentication, transactions, resource fencing, and official
-MongoDB Node.js driver integration.
+Transaction source reads use the owned snapshot plus buffered writes. Foreign-reading and writing aggregate stages reject inside transactions, including nested facets. Their nontransaction behavior is preserved.
 
-Current execution status:
+Finite count pipelines support optional match, optional skip, positive integer limit, and count or the driver's constant-one count group. The query engine owns streaming accumulation and shares the normal aggregation work ceiling for stage dispatch, syntax, specifications, predicates, and BSON payload processing. Physical traversal preserves hints and deadlines. Generic materialization limits and the existing transaction snapshot admission cap of 10,000 documents / 32 MiB remain.
 
-- the legal and provenance gate passed on 2026-08-22
-- Phase 0 and the Phase 1 gate have not passed
-- 5.x maintenance now treats `data.rdb` as authoritative, ignores unverifiable
-  legacy hint contents, streams CRC validation, and repairs only an incomplete
-  final record-header prefix
-- strict latest-per-key runtime WAL recovery and post-publication point-write
-  fencing were released in 5.3.2; crash-safe ordinary DDL, debug-server security,
-  and general crash/soak qualification remain separate required 5.x slices
-- 5.4.0 released the non-serving, fixture-only A0 committed-log kernel with one
-  static timeline, contiguous LSNs, exact retry receipts, bounded
-  change-proportional appends, strict pair recovery, and ambiguity fencing; the
-  follow-up A1/MVCC/rollover/checkpoint work is parked unreleased on the
-  `storage-vnext-a1-wip` branch
-- A0 has no BSON document substrate, MVCC, checkpoint, segment rollover, OpenRaft
-  integration, wire serving, migration, or availability claim; Phase 3B and the
-  earlier Phase 0/1 gates have not passed
-- 5.5.0 serves the truthful single-node concern profile on the legacy engine:
-  fsync-backed `w: 1`/`majority`/`j` acknowledgement, committed-state reads for
-  `local`/`available`/`majority`/`linearizable`, database-wide transaction
-  snapshots pinned by the namespace publication clock with fail-closed
-  `SnapshotUnavailable` retries, cooperative `maxTimeMS` deadlines, and
-  per-session idempotent commit/abort outcomes; unsatisfiable `w: N > 1`, tag
-  sets, cluster-time read concerns, and journaled writes on the memory backend
-  remain rejected
-
-## Product objective
-
-Build SmartDB into a commercially operated, MongoDB-driver-compatible managed
-database service in this order:
-
-1. truthful and provable single-node engine
-2. certified compatibility profile
-3. managed single-node service
-4. replicated high-availability service
-5. whole-database placement and migration
-6. intra-database sharding only when customer scale requires it
-
-Reimplementing MongoDB-compatible server behavior and its persistence engine is the
-fixed strategy. SmartDB owns the wire, semantic, execution, storage, topology, and
-service contracts. Storage VNext does not embed SQLite, redb, Fjall, LMDB, RocksDB,
-or another database engine.
-
-## Legal and provenance gate
-
-The official MongoDB wire-protocol documentation and specifications carry terms
-that expressly restrict commercial database and database-as-a-service adaptation.
-This plan does not interpret those terms. Before expanding compatibility work:
-
-- obtain written legal guidance for interoperability, clean-room implementation,
-  product claims, and trademarks
-- audit the provenance of existing compatibility code and tests
-- classify disputed material for retention, independent rewrite, or quarantine
-- document which behavioral evidence and test sources implementation teams may use
-- market the service under an approved SmartDB identity and compatibility claim
-
-No compatibility feature enters a commercial release without passing this gate.
-
-## Maintenance policy
-
-SmartDB 5.x is the maintenance lane for existing consumers:
-
-- accept security fixes, data-correctness fixes, explicit rejection of unsupported
-  behavior, and bounded operability improvements
-- do not add broad MongoDB surface area or introduce a second durability model
-- keep persisted 5.x behavior supported until a tested offline conversion or
-  snapshot/restore migration into Storage VNext exists
-- release small fixes independently when their invariants and compatibility impact
-  can be verified in isolation
-
-Storage VNext is developed behind explicit format and capability boundaries. It
-must never dual-write through both the legacy and VNext commit paths.
-
-## North-star architecture
-
-### Wire gateway
-
-- parse and encode the declared wire profile exactly
-- validate complete command schemas before acquiring resources or mutating state
-- advertise only capabilities and limits proven by target-driver handshake tests
-- reject every unsupported command, concern, or option explicitly
-
-### BSON semantic core
-
-- own one implementation of BSON equality, ordering, hashing, numeric promotion,
-  null/missing behavior, array traversal, typed identifiers, projections,
-  expressions, and updates
-- feed the same semantics into queries, indexes, transactions, aggregation,
-  uniqueness checks, and change streams
-
-### Planner and executor
-
-- use one planner for find, count, update, delete, aggregate, and explain
-- stream storage-backed cursors instead of materializing complete result sets
-- enforce deadlines, cancellation, admission, and work accounting at every loop
-
-### Transactional storage kernel
-
-- own one authoritative, checksummed, monotonically ordered commit log
-- assign a stable timeline and LSN to every committed mutation
-- commit documents, DDL, catalogs, index definitions, users, retryable-write
-  outcomes, and transaction outcomes through the same atomic boundary
-- serve database-wide MVCC snapshots and deterministic checkpoint-plus-log recovery
-
-### Replication
-
-- replicate the committed Storage VNext log through a reviewed consensus design
-- use consensus terms to fence leaders; allocation fencing controls placement and
-  publication but never substitutes for data-plane consensus
-- derive majority acknowledgement, failover, snapshot installation, causal times,
-  and retry outcomes from the replicated commit history
-
-### Administration and control plane
-
-- keep RBAC-protected database administration such as `currentOp`, `killOp`,
-  `explain`, and user management available through the certified wire surface
-- keep service administration such as provisioning, placement, billing, backup,
-  restore, upgrades, and incident actions on a separate privileged API
-- start with one process per tenant or isolated replica group; shared-process
-  tenancy requires hard per-tenant admission and noisy-neighbor proof
-
-## Phase 0: contract and measurement
-
-- complete the legal and provenance gate
-- select the first exact driver, ODM, topology, command, option, and error profile
-- maintain raw-wire and target-driver handshake tests
-- publish an out-of-band machine-readable compatibility manifest
-- version engine build, wire profile, storage format, and service API independently
-- establish baseline latency, memory, disk amplification, and restart benchmarks on
-  Linux amd64 and arm64
-
-Gate: approved implementation sources and claims, an exact compatibility profile,
-and reproducible baseline measurements.
-
-## Phase 1: truthful single-node behavior
-
-- stop over-advertising MongoDB version, wire behavior, and batch limits
-- serve the single-node concern profile truthfully and reject every concern,
-  option, command, or administrative stub whose guarantee one node cannot
-  provide (completed 2026-09-02: concerns, deadlines, database-wide transaction
-  snapshots, idempotent transaction outcomes)
-- completed slice (2026-08-25): validate OP_MSG required flags, CRC-32C
-  checksums, bounded framing, and exact no-response admission and execution
-- preserve complete BSON type and value identity for document and index keys
-- reconcile ambiguous point-write failures before returning control to callers
-- journal DDL and sync every required file and parent directory
-- distinguish incomplete final records from interior corruption and fail closed
-- isolate or secure all debug and privileged management surfaces
-- add an operation registry, deadlines, cancellation, real `currentOp`/`killOp`, and
-  truthful metrics
-- write acknowledgement, recovery, corruption, and supported-filesystem contracts
-
-Gate: deterministic failpoints and randomized crashes produce no acknowledged-write
-loss, phantom documents, query/storage divergence, or unique-constraint violations
-within the declared single-node fault model.
-
-## Phase 2: conformance foundation
-
-- build a legally approved behavioral comparison harness
-- fuzz raw wire framing, flags, malformed BSON, and command boundaries
-- validate commands into typed internal representations before execution
-- add exact response, error-code, error-label, and side-effect contracts
-- generate semantic corpora across BSON types, boundaries, arrays, null/missing,
-  NaN, Decimal128, projections, paths, expressions, and identifiers
-- prevent any advertised capability from expanding without associated tests
-
-Gate: no known P0/P1 mismatch in the certified profile, and every unsupported path
-rejects before side effects.
-
-## Phase 3A: storage-substrate decision
-
-Decision accepted 2026-08-28: implement a fully SmartDB-owned transactional storage
-kernel and persisted format. The legacy Bitcask engine remains only in the 5.x
-maintenance and offline-conversion lane; it is not the VNext durability boundary.
-Third-party database engines are excluded from the VNext runtime. Libraries for
-bounded primitives such as checksums, compression, or cryptography remain subject
-to ordinary evidence and license review without becoming an embedded database.
-
-The custom kernel and every candidate primitive are evaluated against:
-
-- crash and power-loss behavior
-- atomic batch and MVCC support
-- checkpoint, backup, and replication integration
-- memory, latency, write amplification, and restart behavior
-- format ownership, migration, licensing, and supply-chain risk
-
-Gate: one architecture decision selects exactly one authoritative durability
-boundary. Dual-WAL and dual-commit designs are prohibited.
-
-## Phase 3B: Storage VNext
-
-- add a segmented, checksummed global commit log with stable timelines and LSNs
-- encode atomic data, DDL, catalog, index-metadata, and auth batches
-- persist request deduplication, retryable-write results, and transaction outcomes
-- add database-wide MVCC read timestamps and change-proportional transactions
-- replace full index-engine clones with incremental deltas
-- add immutable checkpoints, bounded replay, and a durable format manifest
-- move compaction to a rate-limited background service
-- add group commit and real batch publication
-- define an offline 5.x migration and N-1 compatibility contract
-
-Gates:
-
-- one-document transaction cost is independent of database size
-- repeated recovery is idempotent and produces the same content digest
-- ENOSPC, EIO, short writes, and failed fsyncs yield success, explicit ambiguity, or
-  fail-closed state, never silent divergence
-- 24-hour nightly and 72-hour release-candidate soaks meet declared memory, disk,
-  and restart ceilings on both Linux architectures
-
-## Phase 4: query, index, and resource model
-
-- implement one streaming planner/executor
-- use canonical order-preserving BSON index keys
-- add range, compound-prefix, and index-ordered top-k scans
-- implement correct multikey expansion, uniqueness, and TTL execution
-- compile predicates and regular expressions once per operation
-- implement positional updates and array-filter semantics
-- add process-wide and per-tenant admission for memory, scans, connections, I/O,
-  index builds, and maintenance
-- add configurable service classes and fair scheduling
-- add only features selected by the certified compatibility profile
-
-Semantic work may proceed after Phase 2. Crash-durable retryable outcomes,
-transaction publication, and durable index publication move onto Storage VNext.
-The single-node concern profile is served today: every acknowledged write is
-durable before the response, reads observe committed state, and transactions
-observe one database-wide snapshot. Multi-voter majority acknowledgement and
-cluster-time read concerns remain rejected until replication implements them.
-
-Gate: bounded-memory scans at maximum declared scale, published p99 latency and
-capacity curves, proven tenant isolation, and zero certified-profile mismatches.
-
-## Phase 5: backup, PITR, and managed single-node service
-
-- create online immutable checkpoints at exact LSNs
-- archive subsequent commit-log segments continuously
-- encrypt, checksum, and sign backup manifests with timeline, format, LSN range,
-  and key identity
-- restore into a new root, replay to an exact LSN or timestamp, verify its content
-  digest, and promote atomically
-- retain logical export/import as migration tools rather than backup primitives
-- add versioned service classes, immutable node placement, node-bound grants,
-  TLS/mTLS endpoints, credential rotation/revocation, quotas, metering, backup
-  policy, maintenance windows, audit, and idempotent asynchronous operations
-- operate one process per tenant initially
-- qualify signed images, SBOM/provenance, canaries, rollback, alerts, runbooks, and
-  security before any general-availability claim
-
-Launch order: internal alpha, private beta, then single-node GA with explicit
-availability, RPO, RTO, capacity, and compatibility limits.
-
-Gate: repeated maximum-size restore drills meet the published RPO/RTO; billing
-reconciles; upgrades and rollback/restore pass; cross-tenant network, auth, quota,
-backup, and restore isolation is proven; no plaintext secret reaches persisted or
-logged state.
-
-## Phase 6: replication and high availability
-
-- replicate the Storage VNext log across three replicas in separate failure domains
-- add terms, quorum commit, snapshot installation, log catch-up, and membership
-  changes
-- extend the single-node concern profile to multi-voter majority acknowledgement
-  and cluster-time read concerns
-- publish accurate `hello` topology and election metadata
-- add causal times, retryable outcomes, idempotent transaction commits, and
-  failover-safe session behavior
-- support mixed N/N-1 rolling upgrades
-- keep one database assigned to one replica group
-
-Majority RPO 0 means a majority-acknowledged write survives loss of any one replica
-within the declared replica-group fault model.
-
-Gate: no split-brain commits under partitions; accepted histories satisfy the
-declared consistency model; leader failure meets the published RTO; and retries
-return the original outcome exactly once.
-
-## Phase 7: managed HA and compatibility expansion
-
-- place complete databases across replica groups
-- migrate a database through checkpoint transfer, log catch-up, and an atomic
-  routing-epoch change
-- permanently fence stale owners
-- add automated replica replacement and maintenance-aware failover
-- derive durable change streams from the replicated commit log
-- certify additional drivers and ODMs
-- expand aggregation, views, indexes, and administrative behavior according to
-  measured workload demand
-- publish machine-generated compatibility and SLO matrices
-
-Gate: no open P0/P1 defect in the certified surface, regional and maintenance
-exercises pass, and customer-visible incident/status processes are operational.
-
-## Phase 8: horizontal scale
-
-Scale complete databases across replica groups first. Add intra-database sharding
-only after single-group limits become a measured customer constraint:
-
-- separate router and replicated placement catalog
-- range ownership and epochs
-- snapshot-plus-log range migration
-- stale-router rejection and permanent owner fencing
-- per-shard replica groups, balancing, and range cleanup
-- cross-shard transactions only as a separately justified program
-
-Gate: migration crashes cannot create gaps, duplicates, or dual writers; stale
-routing metadata cannot mutate old ownership; shard and region chaos meet the
-published SLOs.
-
-## Release qualification
-
-Every applicable release must pass the narrowest relevant set of these gates:
-
-- raw-wire and target-driver handshake tests
-- generated semantic and certified compatibility suites
-- deterministic persistence failpoints and randomized process-kill tests
-- storage/index/content-digest reconciliation after recovery
-- 24-hour nightly and 72-hour release-candidate resource soaks
-- all published Linux and macOS architectures
-- automated backup/restore drills once backup exists
-- partition, failover, and history verification once replication exists
-- supply-chain, provenance, security, upgrade, and rollback checks for managed tiers
-
-No release may advertise a capability, durability level, topology, or operational
-limit that its release qualification did not exercise.
+Regression coverage includes competing sparse/dense indexes, SCIM compound keys, candidate page count/byte boundaries, malformed hints, buffered transactions, foreign-stage rejection, 12,000-document finite probes, sources exceeding 32 MiB, retained resource caps, and shared aggregation work-budget enforcement.

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartdb',
-  version: '5.5.1',
+  version: '5.6.0',
   description: 'A MongoDB-compatible embedded database server with wire protocol support, backed by a high-performance Rust engine.'
 }

package/readme.hints.md

@@ -1,77 +0,0 @@
-# smartdb hints
-
-## Single-Node Concern Profile, Namespace Clock, and Deadlines (5.5.0)
-
-- `rustdb-commands/src/concerns.rs` is the sole owner of `readConcern`, `writeConcern`, and `maxTimeMS` validation. It runs in the router before authentication, sessions, permits, or gates. The profile is stated for one voting node: fsynced acknowledgement satisfies `w: 1`, `w: "majority"`, and `j: true`; `w: N > 1` is `UnsatisfiableWriteConcern` (100), tag sets are `UnknownReplWriteConcern` (79), cluster-time read concerns and misplaced fields are `InvalidOptions` (72), malformed deadlines are `BadValue` (2). `j: true` on `MemoryStorageAdapter` is `BadValue` through `StorageAdapter::journaled_writes()`. The driver sends a transaction's read concern on whichever statement starts the transaction, so a transaction-starting insert may carry one. Never accept a concern by widening an allowlist; every accepted value must map to a guarantee the engine actually provides.
-- `rustdb-commands/src/clock.rs` (`CommandContext::namespace_clock`) is the publication clock. Every in-process publication that changes visible namespace contents records `publish_namespace` while it still holds the namespace write lock (point writes, transaction commit completion, `create`, `drop`, `renameCollection`, `$out`, `$merge`, oplog revert) or `publish_database` (`dropDatabase`, in-process durable replacement through `invalidate_database_runtime`, and cross-process publication observed through `invalidate_database_runtime_for_epoch`). `$out` publishes right after it drops the old contents and again after the refill, so a failed refill still invalidates pinned snapshots. Index DDL does not change documents and does not publish. Adding a new mutation path without a publication is a snapshot-consistency bug.
-- Transactions pin `TransactionState::snapshot_sequence` when they materialize their first namespace. `transactions::load_transaction_docs_locked` requires the caller to hold the namespace write lock (insert/update/delete already do); `load_transaction_docs` acquires it for readers (`find`, `count`, `distinct`). A namespace whose `last_publication` exceeds the pinned sequence fails with `SnapshotUnavailable` (246); the router labels it `TransientTransactionError` so `withTransaction` restarts. This turns lazy per-namespace materialization into one database-wide snapshot without a second data copy; Storage VNext MVCC read timestamps replace the materialized copy itself.
-- Sessions retain `last_outcome` (`TransactionOutcome::Committed`/`Aborted` per transaction number) through `SessionEngine::finish_transaction_if_bound`. `commitTransaction`/`abortTransaction` without an active binding replay that outcome: repeated commit is `ok`, abort-after-commit is `TransactionCommitted` (256), commit-after-abort is `NoSuchTransaction` (251), and an older `txnNumber` on start is `TransactionTooOld` (225). Outcomes are process-local; after a crash the outcome of an in-flight commit is discoverable only from the data, which is why callers must journal their own attempts inside the transaction.
-- `rustdb-commands/src/deadline.rs` holds the `maxTimeMS` deadline in a tokio task-local set by `CommandRouter::route_with_response_policy`. Checks are cooperative: admission, `deadline::wait_bounded` inside `acquire_write_lock`/`acquire_maintenance_write_gate`, `wait_until_deadline` around the maintenance read lease, `check()` after materialization in readers, `check_every` in insert/delete loops, and one final `check()` before `apply_write_batch_with_completion` in commit. Handler futures are not cancellation-safe, so never wrap a handler in `tokio::time::timeout`; add a check at the next safe point instead. `SMARTDB_TEST_OPERATION_DELAY=<command>:<ms>` (test-support builds only) injects a pre-dispatch delay for deterministic expiry tests.
-- Error labels are derived from the response code in `attach_error_labels` so pre-dispatch rejections and handler errors label identically: transaction statements with 112/246/251 carry `TransientTransactionError`; `commitTransaction` with 50 or 64 carries `UnknownTransactionCommitResult`.
-
-## Storage VNext A0 Committed Log
-
-- `rustdb-kernel` is a fixture-only, non-serving A0 durability slice. It is not linked into the released `rustdb` binary and provides no BSON, CRUD, MVCC, concern, deadline, migration, replication, or availability behavior.
-- The immutable `kernel-manifest.sdb` stores format, group, timeline, and fixed segment identity only. It never stores a mutable commit frontier. Complete prepared/commit-marker pairs in `commit-0000000000000001.sdb` are the sole A0 commit authority.
-- Each new command validates and prepares before append, receives the next contiguous nonzero LSN, and appends one `KPR1` prepared frame plus one `KCM1` marker binding the exact prepared bytes. Exact retries return the original LSN/response without appending; fingerprint conflicts, invalid commands, and segment exhaustion consume no LSN or state.
-- Any append or sync error after write admission is ambiguous and permanently fences that handle, including reads and exact retries. Reopen is the only reconciliation path. Reopen syncs every complete recovered segment before exposing state or receipts.
-- Recovery accepts only the declared timeline, contiguous LSNs, valid CRC frames, a matching marker position/hash, valid commands, and one durable record per receipt identity. It may truncate only an incomplete final frame of the expected kind or a complete valid prepared record without a complete marker. Complete corruption, wrong-kind tails, invalid prepared records, holes, duplicates, and marker mismatches fail without changing the segment.
-- The descriptor-safe root/framing primitives are shared by `rustdb-kernel` and `rustdb-replication`. Their common parent-slot/in-root lock namespace is intentional: kernel and Raft roots must be exclusive and are never interchangeable.
-
-## Authoritative File Recovery
-
-- `data.rdb` is the sole runtime source for rebuilding KeyDir. The v1 `keydir.hint` format cannot prove completeness or data-generation identity, so existing regular hint files are tolerated for compatibility and diagnostics but their contents are never loaded as runtime state. New shutdowns, compactions, and v0-to-v1 migrations do not publish hints.
-- Runtime WAL scanning is bounded and strict: exact current headers, strictly increasing record sequences, operation shapes, lengths, CRCs, and nonduplicated commit references to preceding records must validate. Only an incomplete final record-header or commit-marker prefix with the expected magic is truncated. A complete header with a short payload and every checksum, sequence, magic, or interior failure stop startup without repairing either `data.rdb` or the WAL.
-- Recovery considers only the highest-sequence WAL record for each document key. If that final record is committed, no older markerless intent is replayed; if it is markerless, only that record is replayed. This prevents an older update or delete from overtaking a later acknowledged update or reinsert.
-- A point mutation whose data record reached `data.rdb` and fsynced before its WAL commit marker failed is successful so command indexes and OpLog publish the same mutation. The collection is then sticky-fenced: later document/index access and every write, batch publication, cache invalidation, rename, drop, and database replacement touching it fail until restart; maintenance may skip the collection but must never evict its fence. Pre-publication WAL/data append failures may continue only after both files are durably rolled back to their previously validated lengths; an unverified rollback fences instead.
-- `KeyDir::build_from_data_file()` streams every payload through CRC32 in fixed-size chunks before allocating its key and never allocates the declared value length. It may classify only a final prefix shorter than `RECORD_HEADER_SIZE` with matching record-magic bytes as an incomplete tail. A full header with a short payload and every checksum, magic, length, or UTF-8 key failure are corruption and must leave `data.rdb` unchanged.
-- Compaction durably removes any legacy hint and syncs the collection directory before replacing `data.rdb`. This ordering forces SmartDB 5.2.1 and earlier to rebuild after downgrade even when the compacted file happens to have the hint's old recorded size.
-- Startup maintenance may skip a publication-held database. Its first later storage access still performs the authoritative scan and fails unchanged on corruption; never infer validation of held data from listener startup alone.
-- Every cancellation-unsafe maintenance, fencing, auth, or storage worker must own a clone of the retained storage-root capability through its final file/lock cleanup. Provider installation is a finite joined startup sweep; periodic maintenance starts only after listener setup has no remaining fallible step. `close()` joins maintenance, `RustDb::drop()` requests cooperative shutdown, and Unix socket cleanup is bound to the exact device/inode created by `bind()`.
-
-## Stopped Storage-Root Relocation
-
-- `LocalSmartDb.relocateStoppedStorageRoot()` is the only supported path-changing operation. It uses a dedicated serialized mutating sidecar runner; do not route it through the disposable offline-inspection helper. A bridge whose termination is unconfirmed remains in the static cleanup gate, and no later relocation may spawn until cleanup succeeds.
-- Rust owns the descriptor-relative operation in `rust/crates/rustdb/src/storage_root_relocation.rs`. It never constructs `RustDb`. The fixed journal phases are `preparing`, `prepared`, `exchanged`, `providerPublished`, `sentinelPublished`, and `cleaned`; retries reconcile the exact source-root/destination-absent-or-receipt topology or source-receipt/destination-root topology before continuing.
-- The destination receipt is published with an exclusive rename, then exchanged with the source root using an atomic swap (`RENAME_NOREPLACE`/`RENAME_EXCHANGE` on Linux and `RENAME_EXCL`/`RENAME_SWAP` on macOS). Provider and sentinel replacement also uses checked exchanges, retaining each previous object in the workspace until the replacement is durable and verified. The original root inode ends at the destination and the original receipt inode remains at the source indefinitely. Parent, root, internal, lock, provider, and sentinel identities are revalidated around publication.
-- Eligibility is intentionally narrow: Linux on the existing supported local-filesystem set or macOS on qualified local APFS, exact current owner/provider/sentinel metadata, no auth path digest, and no active resource receipt, allocation, database-mutation workspace, unrelated marker, symlink, unknown entry, or unsafe ownership/mode/link profile. A transaction workspace is eligible only when `rustdb-storage` validates its exact `cleaned` internal journal and matching database transaction marker; workspace debris, unfinished phases, mismatches, and allocation bindings remain unsupported. Empty retired legacy locks are eligible only with the canonical resource-digest/inode name parsed by `resource_fencing`, within its existing bound, and tied to one of those terminal transaction resources. macOS additionally rejects case, Unicode, mount, and ancestry aliases by retained physical identity. Missing metadata is rejected and never initialized by relocation.
-- Both TypeScript startup paths check for `.__rustdb_internal/storage-root-relocation` before `StorageMigrator`. Rust checks immediately after securely opening an existing root and repeats the descriptor-relative check after acquiring the owner lock, before storage initialization. This closes the interval where a relocation can create its workspace and release the lock after startup's first inspection. Occupancy or inspection ambiguity is a startup error; only an exact relocation retry performs recovery.
-- Completed roots retain the SmartDB 5.0.1 provider/sentinel JSON shape, changing only their storage-root path digest. Interrupted Linux state requires 5.1.0 or newer; interrupted macOS state requires a macOS-capable release. Exact replay lasts only while the destination remains current; a later relocation supersedes the old source/destination tuple.
-
-## Restricted Deny-All Views
-
-- `rustdb-commands/src/views.rs` is the sole owner of `system.views` parsing, bounds, canonical `_id_` metadata validation, exact deny-all pipeline validation, and authoritative namespace classification. There is intentionally no mutable cross-command view cache; aggregate and transaction commit reuse one immutable authoritative catalog per database for that operation.
-- The only accepted definition is `viewOn: <same-db collection name>` plus exact pipeline `[{ $match: { $expr: { $eq: [1, 0] } } }]`. A logical view has no physical collection and never reads `viewOn`.
-- `system.views` is a physical reserved catalog. It is visible as a collection in wire `listCollections`, export, import, and content digest, but command-layer `IndexEngine` creation and direct wire access are forbidden. Internal view create/drop uses storage point insert/delete and preserves the storage adapter's canonical `_id` metadata.
-- Catalog loading is bounded to 1,024 view records and 1 MiB of encoded view documents and never enumerates all physical collections. Runtime validation checks each bounded logical name with `collection_exists`; snapshot/import validation uses its already-provided physical-name set without adding a runtime collection ceiling. Exact document shape is `_id`, `viewOn`, and `pipeline`; malformed records, noncanonical indexes, duplicate/reserved names, and physical/view collisions fail closed.
-- Publication permits validate once per newly observed publication state; databases without an epoch use an internal observed sentinel. Epoch refresh validates before recording the new observation. Startup validates databases whose publication lease can be acquired before listener bind; publication-held databases are validated on their first later permit. Snapshot/export explicitly revalidate. Import validates the source candidate, remaps only the exact source database prefix in catalog `_id` values, then validates the target candidate before mutation and clears its observed state after replacement.
-- Content digest hashes the physical catalog but removes its already-validated deterministic database prefix from `_id` first, preserving the database-name-independent digest contract across cross-name imports.
-- DDL and transaction commit take the maintenance write gate. View create/drop additionally lock the logical target and `${db}.system.views` in sorted order. View DDL validates the complete prospective catalog and performs cursor/index cleanup before its final point insert/delete, so cancellation cannot publish catalog state without cleanup. Commit locks every touched namespace in sorted order, loads one catalog per touched database, reclassifies each namespace, and transfers those guards into the storage completion closure.
-- Mutation handlers reclassify after acquiring their target namespace lock and before existence, auto-create, or lazy-index work. Logical-view data/index/output/rename mutations and direct `system.views` operations return `CommandNotSupportedOnView` code 166; `dropCollection` removes a logical view, while direct catalog drop remains forbidden. Ordinary create collision with a logical view returns `NamespaceExists`.
-- Find/count/distinct are empty outside a transaction, aggregate applies the client pipeline to an empty source, and lookup/union resolution is empty. A transaction first reading a current view snapshots an empty vector, while a populated snapshot captured before a rename-plus-view transition remains authoritative for find/count/distinct until commit revalidation. Do not infer view semantics from physical collection absence.
-- Successful create/drop/rename/dropDatabase publication removes affected namespace cursors before releasing DDL guards. A subsequent `getMore` cannot continue against stale namespace contents.
-- Management oplog revert rejects entries targeting `system.views` or a currently logical view. Logical-view `dropCollection` and whole-database drop are the catalog-removal exceptions; direct `system.views` drop is code 166. Database drop clears database indexes, cursors, oplog state, and observed publication state.
-
-## Multikey Query Planning
-
-- SmartDB detects when an indexed path resolves through an array and keeps that state on the in-memory index until a complete rebuild.
-- Array-backed indexes are maintained for catalog and constraint compatibility but are not used to narrow queries; reads fall back to authoritative scans until full MongoDB multikey key expansion is implemented.
-- A scalar-to-multikey transition invalidates an in-progress paged index read with a write conflict; retrying starts with the authoritative scan path instead of returning a partial result.
-- Index data and multikey state are rebuilt from authoritative documents at startup, so this behavior requires no persisted-data migration.
-
-## No-Op Write Detection (2.14.0)
-
-- `update`/`findAndModify` post-images that equal the stored document skip storage, WAL, index staging, and oplog entirely; volatile-only rewrites (top-level `_updatedAt` differs, everything else equal) are skipped the same way and keep the stored `_updatedAt`. Both report `matched=1, modified=0`.
-- Classification lives in `rustdb-commands/src/handlers/update_handler.rs` (`classify_document_rewrite`); the volatile field list is the `VOLATILE_METADATA_FIELDS` const. The comparison is key-order-insensitive (replacement paths re-append `_id`) and bails to "changed" on duplicate keys.
-- Counters: `serverStatus.writes.{updatesWritten, noopSkipped.identical, noopSkipped.volatileOnly}` backed by `WriteMetrics` on `CommandContext`. Wire-level coverage in `test/test.noop-writes.ts`.
-- Consequence for consumers: a save that changes nothing no longer bumps `_updatedAt` and emits no oplog event. To force a touch, change a non-volatile field.
-
-## Static Rust Binaries
-
-- The Linux Rust binaries in `dist_rust/` are statically linked (static-pie) via `"static": true` in the `@git.zone/tsrust` block of `.smartconfig.json` (tsrust >= 1.4.1). They run on both glibc (Debian/Ubuntu) and musl (Alpine) systems. The macOS amd64 and arm64 binaries use the normal Apple platform linkage.
-- tsrust injects `RUSTFLAGS="-C target-feature=+crt-static"` only into its own per-target cargo invocations and verifies the result (no `PT_INTERP` ELF header). Keep `rust/.cargo/config.toml` free of `rustflags` entries: the injected env variable would replace them, and a repo-wide `rustflags` would also break plain `cargo test`/`cargo check` (proc-macros cannot build with `+crt-static` on linux-gnu without an explicit `--target`). The repo previously carried such `rustflags` lines; they were removed when the tsrust `static` option was adopted.
-- `pnpm run test:rust` runs the Rust unit tests; plain `cargo test` in `rust/` works as well.
-- Verify linkage manually with `ldd dist_rust/<binary>_linux_amd64` → "statically linked".
-- Do NOT verify feature presence in release binaries with `strings`/`grep`: on x86_64 release builds LLVM embeds short string literals (≤16 bytes, e.g. BSON doc keys like `volatileOnly`) as instruction immediates, so they are invisible to byte scans while the aarch64 binary shows them in `.rodata`. This produces false "stale binary" alarms. Verify behavior by running the binary (the wire tests spawn the host-arch binary from `dist_rust/`), or disassemble: `objdump -d <bin> | grep <hex-of-literal-LE>`.
-- Release builds use the tsrust 1.10 native matrix. The enabled gitzone test preflight checks the exact project-local tool, cross-execution, worker identities, toolchains, and remote root before versioning. The post-version build bundles TypeScript, transfers exact HEAD into isolated local and SSH workers, installs the frozen lockfile, executes all four target binaries, retrieves bounded artifact sets, and transactionally publishes the complete provenance matrix. Remote cleanup is limited to the unique owner-marked child below `SMARTDB_RELEASE_MACOS_TEMP_ROOT` and gates publication.
-- Workspace release-age exclusion arrays replace operator-global exclusions, and matrix workers use synthetic homes with a separate age policy. For immediate dependency-chain releases, enumerate every required exact audited package version across all install contexts; do not disable the age gate or widen an exception to a scope.