@harperfast/harper 5.2.0-alpha.4 → 5.2.0-alpha.6

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@harperfast/harper",
 	"description": "Harper is an open-source Node.js performance platform that unifies database, cache, application, and messaging layers into one in-memory process.",
-	"version": "5.2.0-alpha.4",
+	"version": "5.2.0-alpha.6",
 	"license": "Apache-2.0",
 	"homepage": "https://harper.fast",
 	"bugs": {
@@ -56,6 +56,7 @@
 		"build:watch": "npm run build -- --watch --incremental",
 		"typecheck": "tsc --project tsconfig.json",
 		"typecheck:fast": "npx -y -p typescript@7.0.2 tsc --noEmit --project tsconfig.json",
+		"test:types": "tsc --project unitTests/types/tsconfig.json",
 		"package": "./build-tools/build.sh",
 		"lint": "oxlint --format stylish --deny-warnings .",
 		"lint:required": "oxlint --format stylish --quiet .",
@@ -166,7 +167,7 @@
 	},
 	"dependencies": {
 		"@aws-sdk/client-s3": "^3.1012.0",
-		"@aws-sdk/lib-storage": "3.1076.0",
+		"@aws-sdk/lib-storage": "3.1080.0",
 		"@endo/static-module-record": "^1.1.2",
 		"@fastify/autoload": "^6.3.1",
 		"@fastify/compress": "^8.3.1",
@@ -174,6 +175,7 @@
 		"@fastify/static": "^9.1.3",
 		"@harperfast/extended-iterable": "^1.0.1",
 		"@harperfast/rocksdb-js": "^2.4.0",
+		"@harperfast/skills": "^1.10.8",
 		"@turf/area": "6.5.0",
 		"@turf/boolean-contains": "6.5.0",
 		"@turf/boolean-disjoint": "6.5.0",
@@ -198,7 +200,7 @@
 		"fast-glob": "3.3.3",
 		"fastify": "^5.8.2",
 		"fastify-plugin": "^5.1.0",
-		"fs-extra": "11.3.5",
+		"fs-extra": "11.3.6",
 		"graphql": "^16.10.0",
 		"graphql-http": "^1.22.4",
 		"gunzip-maybe": "1.4.2",

package/resources/DESIGN.md

@@ -25,6 +25,9 @@ See also: `../DESIGN.md` for cross-cutting non-obvious internals (RecordObject p
 | `auditStore.ts`         | Append-only audit log records                                                                                                        |
 | `nodeIdMapping.ts`      | Maps node IDs ↔ timestamps for replication ordering                                                                                  |
 | `openApi.ts`            | Generates OpenAPI/JSON Schema from `@export` schemas                                                                                 |
+| `defineTable.ts`        | Code-first table authoring (`defineTable` + `types`) — a TS front-end to the canonical `table()` model                               |
+| `defineResource.ts`     | Per-method request contract (`defineResource` / `Resource.withSchema`, `t`, `schemaOf`) — typed handlers + edge validation           |
+| `jsonSchemaTypes.ts`    | Shared `JsonSchemaFragment` IR + `attributeToFragment` projector (one vocabulary for validation/OpenAPI/MCP)                         |
 | `analytics/`            | Telemetry recording (separate from monitoring)                                                                                       |
 
 ---
@@ -75,21 +78,22 @@ One giant `makeTable()` factory that returns a `TableResource extends Resource`
 
 ## "Where is X" cheat sheet
 
-| Question                                            | Where                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| How is a CRUD request authorized?                   | `Table.ts → #section: authz-hooks`; defaults in `Resource.ts` (`allowRead` etc.)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| Where does versioning / conflict resolution happen? | `Table.ts → _writeUpdate` (`#section: write-path-internals`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| How does `search()` choose an index?                | `Table.ts → search` (`#section: search-query`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
-| How are subscriptions replayed?                     | `Table.ts → subscribe` (`#section: pub-sub`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| How is the response body shaped (select clause)?    | `Table.ts → transformEntryForSelect` (`#section: search-query`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
-| Where is record-level TTL evaluated?                | `Table.ts → setTTLExpiration` (`#section: lifecycle-admin`); `Updatable.getExpiresAt` (`#section: setup-and-factory`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| How are residencies enforced (replication)?         | `Table.ts → #section: lifecycle-admin` (residency block: `getResidencyRecord`, `setResidency`, `setResidencyById`, `getResidency`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
-| How is the RecordObject prototype applied?          | `RecordEncoder.ts` (see `../DESIGN.md`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| Where is the per-request transaction stored?        | `transaction.ts` + `contextStorage` (AsyncLocalStorage)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| How does a query opt out of a read snapshot?        | Pass `snapshot: false` on the search request (e.g. `get_analytics`). `Table.ts → search` calls `txn.useReadTxn(snapshot === false)`; on RocksDB `DatabaseTransaction.getReadTxn` then builds the read txn with `{ disableSnapshot: true }` so a long scan reads latest without pinning a snapshot. No-op on LMDB (`LMDBTransaction.useReadTxn`).                                                                                                                                                                                                                                                                             |
-| How does a URL path map to a Resource?              | `Resources.ts → getMatch` (exact/prefix fast path) then `matchParamRoute` (parameterised routes); see "Path routing" below                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
-| How does HNSW keep the graph connected on delete?   | `indexes/HierarchicalNavigableSmallWorld.ts → index()` delete path: zero-degree orphans reindexed via `needsReindexing`; severed multi-node islands detected and reconnected by `repairSeveredNeighbors` (#1712)                                                                                                                                                                                                                                                                                                                                                                                                             |
-| How is a filter applied _during_ a vector search?   | Predicate-aware traversal (#1241): `search.ts → executeConditions` composes companion AND conditions + a request `vectorFilter` + a static `allowReadRecord` RBAC hook into one `(primaryKey) => boolean` (`composeRecordFilter`) and passes it to `HierarchicalNavigableSmallWorld.search(cond, ctx, filter)`. The filter gates result admission at layer 0 only (routing ignores it, ACORN-style); a visit budget (`filterExpansion`) bounds the under-filled/selective case. Very selective _condition_ filters are instead diverted to the exact brute-force path by the query planner's `estimateCountAsSort` ordering. |
+| Question                                            | Where                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| How is a CRUD request authorized?                   | `Table.ts → #section: authz-hooks`; defaults in `Resource.ts` (`allowRead` etc.)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| Where does versioning / conflict resolution happen? | `Table.ts → _writeUpdate` (`#section: write-path-internals`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| How does `search()` choose an index?                | `Table.ts → search` (`#section: search-query`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+| How are subscriptions replayed?                     | `Table.ts → subscribe` (`#section: pub-sub`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| How is the response body shaped (select clause)?    | `Table.ts → transformEntryForSelect` (`#section: search-query`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| Where is record-level TTL evaluated?                | `Table.ts → setTTLExpiration` (`#section: lifecycle-admin`); `Updatable.getExpiresAt` (`#section: setup-and-factory`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| How are residencies enforced (replication)?         | `Table.ts → #section: lifecycle-admin` (residency block: `getResidencyRecord`, `setResidency`, `setResidencyById`, `getResidency`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+| How is the RecordObject prototype applied?          | `RecordEncoder.ts` (see `../DESIGN.md`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| Where is the per-request transaction stored?        | `transaction.ts` + `contextStorage` (AsyncLocalStorage)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| How does a query opt out of a read snapshot?        | Pass `snapshot: false` on the search request (e.g. `get_analytics`). `Table.ts → search` calls `txn.useReadTxn(snapshot === false)`; on RocksDB `DatabaseTransaction.getReadTxn` then builds the read txn with `{ disableSnapshot: true }` so a long scan reads latest without pinning a snapshot. No-op on LMDB (`LMDBTransaction.useReadTxn`).                                                                                                                                                                                                                                                                                                                                                                      |
+| How does a URL path map to a Resource?              | `Resources.ts → getMatch` (exact/prefix fast path) then `matchParamRoute` (parameterised routes); see "Path routing" below                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
+| How does HNSW keep the graph connected on delete?   | `indexes/HierarchicalNavigableSmallWorld.ts → index()` delete path: zero-degree orphans reindexed via `needsReindexing`; severed multi-node islands detected and reconnected by `repairSeveredNeighbors` (#1712)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| How is a filter applied _during_ a vector search?   | Predicate-aware traversal (#1241): `search.ts → executeConditions` composes companion AND conditions + a request `vectorFilter` + a record-scoped `allowRead` override into one `(primaryKey) => boolean` (`composeRecordFilter`) and passes it to `HierarchicalNavigableSmallWorld.search(cond, ctx, filter)`. The filter gates result admission at layer 0 only (routing ignores it, ACORN-style); a visit budget (`filterExpansion`) bounds the under-filled/selective case. Very selective _condition_ filters are instead diverted to the exact brute-force path by the query planner's `estimateCountAsSort` ordering.                                                                                          |
+| How is row-level read access control enforced?      | Unified `allowRead` (#1422 gap 2 / #1241): an application-OVERRIDDEN `allowRead` (detected via the `isDefaultAllowRead` marker on the framework defaults) is record-scoped — evaluated once per record with `this` = the (frozen) record during query execution, fail-closed on throw, dispatched via the resolved method (never `record.allowRead` lookup — data shadowing). The authorize wrapper (`Resource.ts → authorizeActionOnResource`) defers collection reads on tables (`supportsRowLevelAllowRead`) to this per-record path; single-record `get` keeps the entry check (record loaded, proxied reads work). Records also expose a non-enumerable `allowRead` delegate on the per-table `structPrototype`. |
 
 ---
 
@@ -128,6 +132,29 @@ Tests: `../unitTests/resources/paramRoutes.test.js` (unit) and `../integrationTe
 
 ---
 
+## Typed, discoverable resources (code-first schema + request contract)
+
+Design record: the full RFC and its type-level design proofs live in the design PR (**HarperFast/harper#1503**); this section is the retained summary.
+
+**The principle.** Harper strips TypeScript at runtime (`--conditions=typestrip`), which erases types and rules out metadata-emitting decorators. So **runtime metadata must be values, and TypeScript types are _derived_ from those values — never the reverse.** Everything here is erasable syntax; the values survive stripping, the types are inferred, nothing can drift. One shared IR — `JsonSchemaFragment` (`jsonSchemaTypes.ts`), produced by `attributeToFragment` — feeds validation, OpenAPI, and MCP, so those surfaces cannot silently disagree.
+
+**Code-first tables (`defineTable.ts`).** `defineTable(name, shape, opts)` authors a table in TypeScript and eagerly registers it through the same `table()` factory GraphQL drives — the returned value _is_ the live table class (`Track.get/put/...` work, `new Track()`/`instanceof` hold). Fields come from the `types` vocabulary (getter flags: `string.indexed`, `id.primaryKey`, `date.createdTime`); per-verb shapes are inferred projections discoverable as members (`(typeof Track)['$insert' | '$upsert' | '$patch' | '$query' | '$record']`). Relationships use lazy thunks (`types.relation(() => Album, { from })`) so forward references/cycles resolve at query time; `relationOf`/`hasManyOf` are the escape hatch for a mutual pair whose eager const-inference would otherwise collapse to `any`.
+
+**Per-method request contract (`defineResource.ts`).** Two front-ends, same runtime metadata:
+
+- `defineResource(contract, impl)` — function form (an object of verb handlers).
+- `Resource.withSchema(contract)` — class form; `extends` it and implement the declared verbs. It pins `static loadAsInstance = false` so instance verbs receive the converged `(target, data)` arg order the types assume (the default dispatch order is `(data, target)` — see `Resource.post/put/patch`).
+
+A contract is `{ path, record?, get/post/put/patch/delete: { query?, body?, response? } }`. Handler types are **derived** from it: path params from a template-literal parse of `path`, query/body/response from the schema's inferred type. It is a **subset, not a fork** — a handler gets the SAME `RequestTarget`, structurally narrowed (`target.id: string`, `target.get('expand')` typed by the query schema), and the resource still registers/serves like a plain one. The narrowed types are justified by **runtime enforcement**: each declared verb validates/coerces `query`/`body` before dispatch and throws a structured 400 (`ValidationError`, per-field `{ path, code, message }[]`) — the same bargain `Table.validate` makes for tables.
+
+**Vocabulary.** The built-in `t` (`t.string/number/integer/boolean/date/enum/array/object`) and `schemaOf<T>(source?)` both reduce to `JsonSchemaFragment`. A `defineTable` projection slots straight into a contract body/response — `schemaOf<(typeof Track)['$insert']>({ table: Track, projection: 'insert' })` derives the compile-time type from the projection and the runtime fragment from the table's attributes (via `projectTableFragment` → `attributeToFragment`). **Nullability:** non-nullable by default (a bare `t.string` rejects `null`); `.optional` allows absence, `.nullable` allows an explicit `null`; table-derived bodies mirror `Table.validate`'s policy (null rejected only when `nullable === false`).
+
+**Downstream surfaces.** `openApi.ts` emits a contract's query params, request body, and response for parameterised routes; `components/mcp/tools/application.ts` drives the tool input/output schema off the contract and binds arbitrary path params + query (`applyContractInputs`), which lifts the generated-verb binding restriction for contract resources. `ValidationError` (`../utility/errors/hdbError.ts`) extends `ClientError` (400) so existing HTTP handling is unchanged; the structured issues ride on `.detail`/`.errors`.
+
+Tests: `../unitTests/resources/defineResource.test.js`, `../unitTests/resources/defineTable-registration.test.js`, `../unitTests/resources/openApi-contract.test.js`, `../unitTests/components/mcp/tools/application-contract.test.js`.
+
+---
+
 ## Conventions
 
 - **Never** remove `transactional()` from a static method on `Resource` — it owns transaction context lifetime.

package/resources/Resource.ts

@@ -18,8 +18,10 @@ import { transaction, contextStorage } from './transaction.ts';
 import { parseQuery } from './search.ts';
 import { RequestTarget } from './RequestTarget.ts';
 import { when, promiseNormalize } from '../utility/when.ts';
+import { logger } from '../utility/logging/logger.ts';
 import { registerLiveSubscription } from '../server/liveSubscriptionAuth.ts';
 import type { JsonSchemaFragment } from './jsonSchemaTypes.ts';
+import { makeSchemaClass, type Contract, type SchemaClass } from './defineResource.ts';
 
 const EXTENSION_TYPES = {
 	json: 'application/json',
@@ -59,6 +61,26 @@ export class Resource<Record extends object = any> implements ResourceInterface<
 	static outputSchemas?: { [verb: string]: JsonSchemaFragment };
 	static mcp?: { annotations?: { [verb: string]: { [key: string]: unknown } } };
 	static hidden?: boolean;
+	/** The per-method request contract, when this resource was built via `withSchema`. */
+	static requestContract?: Contract;
+	/** Per-verb `{ query?, body? }` input schemas derived from the request contract, read by OpenAPI/MCP. */
+	static inputSchemas?: { [verb: string]: { query?: JsonSchemaFragment; body?: JsonSchemaFragment } };
+
+	/**
+	 * Declare a per-method request contract and get back a Resource subclass typed by it. Extend the
+	 * returned class and implement the declared verbs; handlers receive the SAME
+	 * `RequestTarget`, structurally narrowed (path params typed, `target.get(param)` typed by the query
+	 * schema). Each declared verb validates/coerces `query`/`body` before dispatch and returns a
+	 * structured 400 on failure, which is what justifies the narrowed types. The contract also feeds
+	 * OpenAPI and MCP from the one declaration.
+	 */
+	static withSchema<Base extends new (...args: any[]) => any, const C extends Contract>(
+		this: Base,
+		contract: C
+	): SchemaClass<Base, C> {
+		return makeSchemaClass(this, contract);
+	}
+
 	constructor(identifier: Id, source: any) {
 		this.#id = identifier;
 		const context = source?.getContext ? (source.getContext() ?? null) : undefined;
@@ -283,6 +305,19 @@ export class Resource<Record extends object = any> implements ResourceInterface<
 
 	static query = transactional(
 		function (resource: any, query: RequestTarget, _request: Context, data: any) {
+			// On the table path, search(target) uses `data` (the request body carrying the conditions) as
+			// its sole target, while checkPermission was set on the URL `query`. Thread it across so
+			// Table.search sees the flag and enforces row-level allowRead — otherwise a QUERY on a table
+			// with an overridden allowRead returns the full unfiltered set (the deferral skips the entry
+			// check on the promise that search consumes checkPermission, which it never sees).
+			// checkPermission is framework-owned: `data` is the client-controlled QUERY body, so it must
+			// always be overwritten here (not just filled when nullish) — otherwise a client could send
+			// `checkPermission: false` in the body to disable Table.search's row-level allowRead guard.
+			if (resource.constructor.loadAsInstance !== false && data && typeof data === 'object') {
+				const checkPermission = (query as any)?.checkPermission;
+				if (checkPermission == null) delete (data as any).checkPermission;
+				else (data as any).checkPermission = checkPermission;
+			}
 			return resource.search
 				? resource.constructor.loadAsInstance === false
 					? resource.search(query, data)
@@ -491,6 +526,26 @@ export class Resource<Record extends object = any> implements ResourceInterface<
 
 _assignPackageExport('Resource', Resource);
 
+// Warn once per table when an ASYNC allowRead override is used on a row-level-capable table: it is
+// evaluated only at the collection-scope entry check and gets NO per-record/per-event filtering, so
+// the collection-permissive "open, filter per record" pattern would fail open. Row-level filtering
+// requires a synchronous override.
+const asyncAllowReadWarned = new Set<string>();
+function warnAsyncAllowReadOnce(tableName: string | undefined): void {
+	const key = tableName ?? '<anonymous>';
+	if (asyncAllowReadWarned.has(key)) return;
+	asyncAllowReadWarned.add(key);
+	logger.warn?.(
+		`allowRead override on table "${key}" is async: it is evaluated only at request entry (collection scope) with NO per-record or per-event filtering. Row-level (record-scoped) authorization requires a synchronous allowRead — an async collection-permissive override would return unfiltered results.`
+	);
+}
+
+// Mark the built-in allowRead so the authorization flow can tell a framework default from an
+// application override. An overridden allowRead on a table is evaluated per RECORD during query
+// execution (#1422 gap 2 / #1241) — with `this` being each record — instead of once at collection
+// entry where `this` has no record to inspect. Table.ts marks its table-level default the same way.
+(Resource.prototype.allowRead as any).isDefaultAllowRead = true;
+
 export function snakeCase(camelCase: string) {
 	return (
 		camelCase[0].toLowerCase() +
@@ -733,6 +788,43 @@ function transactional(
 			}
 			if (checkPermission) {
 				if (loadAsInstance !== false) {
+					// Does this table carry an APPLICATION-overridden allowRead (record-scoped, #1422 gap 2)?
+					// The framework defaults are marked isDefaultAllowRead; anything else is an app override.
+					// Row-level authorization (per-record query traversal, per-event delivery) is SYNC-only.
+					// A SYNC application override participates in it; an ASYNC override cannot (traversal
+					// and delivery can't await), so it is evaluated ONLY at the collection-scope entry
+					// check — a table/connection-level decision, the same as before this feature. It must
+					// therefore make a complete decision at collection scope; the collection-permissive
+					// "open, filter per record" pattern requires a synchronous override (warned below).
+					const overridden =
+						options.type === 'read' &&
+						(resource.constructor as any)?.supportsRowLevelAllowRead &&
+						!(resource.allowRead as any)?.isDefaultAllowRead;
+					const rowLevelOverride = overridden && (resource.allowRead as any)?.constructor?.name !== 'AsyncFunction';
+					if (rowLevelOverride) {
+						// Durable signal that this read was authorization-checked, for the subscription
+						// delivery filter (the entry check below clears query.checkPermission before
+						// Table.subscribe runs, and an anonymous-but-checked subscription has no user to key on).
+						(query as any).rowLevelAuthChecked = true;
+					} else if (overridden) {
+						warnAsyncAllowReadOnce((resource.constructor as any)?.name);
+					}
+					// Deferral (per-record enforcement instead of a meaningless collection-scope entry
+					// verdict): `get` with a non-null id is a true single-record read (entry check runs with
+					// the record loaded) and only defers when isCollection — mirroring the isSearchTarget
+					// routing. ALL search/query calls defer: a present id there is a starts_with/prefix SEED
+					// (multi-record scan), so an entry verdict would gate — or grant — the whole scan; both
+					// invoke instance search() directly, which consumes checkPermission and arms the guard.
+					// subscribe/connect are NOT deferred: the entry check is the connection grant, and an
+					// override there may be connection-level (e.g. an MQTT topic ACL that decides on
+					// context.topic, not record fields, and must run at subscribe time); a record-level
+					// override returns permissive at collection scope to open, and Table.subscribe then
+					// filters each delivered event per record (#1419).
+					if (rowLevelOverride && !isSubscribeAction && (options.method !== 'get' || query.isCollection)) {
+						return when(data, (data) => {
+							return runAction(data);
+						});
+					}
 					// do permission checks, with allow methods
 					let allowed;
 					try {
@@ -802,8 +894,13 @@ function registerLiveSubscriptionForContext(subscription: any, resource: any, qu
 			// and getCurrentUser() (which reads the resource's context) — evaluate against current state,
 			// not the stale user captured at subscribe time.
 			if (context) (context as any).user = fresh;
-			// Re-run the same table/RBAC-level allowRead the subscription was granted with, against the
-			// fresh user. No per-record evaluation — this matches how access was originally granted.
+			// Re-run the SAME allowRead the subscribe entry check ran, against the fresh user — this
+			// mirrors the connection grant exactly. For a record-scoped override (#1419), that override
+			// gated the connection at collection scope (typically composing the table/RBAC grant via
+			// `super.allowRead`), and its per-record decisions are enforced separately during delivery;
+			// re-running it here re-verifies whatever it gated the connection on — a connection-level
+			// override (e.g. an MQTT topic ACL) re-checks its topic grant, and a record-scoped override
+			// that composes `super` re-checks the RBAC baseline. No per-record evaluation here.
 			const reTarget: any = new RequestTarget();
 			reTarget.id = capturedId;
 			reTarget.isCollection = capturedIsCollection;