@haverstack/core 0.14.0 → 0.16.0

package/dist/stack.js

@@ -13,13 +13,13 @@
  *
  * Apps should never talk to a StackAdapter directly.
  */
-import { generateId, isValidIdFormat, idTimestamp } from './id.js';
+import { generateId, generateIdForTimestamp, isValidIdFormat, idTimestamp, MAX_ID_TIMESTAMP, } from './id.js';
 import { hashSchema, isCompatible, parseTypeId, baseIdOf, diffSchemas, isWellFormedTypeId, } from './schema.js';
 import { validateContent, validateReservedKeys } from './validate.js';
 import { applyMergePatch } from './merge.js';
 import { checkAccess, groupRoleFromAssociations, validatePermissions } from './access.js';
 import { firstRecordedAttachment } from './attachment-download.js';
-import { ChangeEmitter, RelayDelivery, Subscription, buildEmission, matchesFilter, } from './changes.js';
+import { ChangeEmitter, RelayDelivery, Subscription, buildEmission, matchesFilter, passesUnlistedBoundary, } from './changes.js';
 import { SYSTEM_TYPES, GRANT_ACTIONS } from './types.js';
 // -------------------------------------------------------
 // Supporting types
@@ -539,19 +539,76 @@ function validateRecordId(id) {
     }
 }
 /**
- * Timestamp-prefix plausibility check for grantee-minted IDs
- * (ScopedStack.create()) — a grantee is untrusted and could otherwise mint
- * an ID that forges its sort position. Pass null to disable.
+ * Validity and range check for the backdating options on unscoped
+ * Stack.create(). A Date is only as good as what the caller parsed it
+ * from, and the two failure modes both need catching here rather than
+ * downstream:
+ *
+ * - **Invalid Date** (`new Date('13/45/2020')` off a malformed import row)
+ *   has a NaN getTime(), and every comparison against NaN is false — so an
+ *   unchecked Invalid Date passes the updatedAt/createdAt ordering check
+ *   and the id/createdAt skew check by turning them off, mints the
+ *   epoch-zero ID `000000000xxx`, and persists a record whose
+ *   `createdAt.toISOString()` throws RangeError in serializeRecord() —
+ *   making that record, and any wire response containing it,
+ *   permanently unreadable.
+ * - **Out of encodable range** — before 1970 crockford32Encode() throws a
+ *   bare RangeError from deep inside the ID encoder, and past
+ *   MAX_ID_TIMESTAMP the derived ID silently grows to 13 characters and
+ *   fails isValidIdFormat().
+ *
+ * Checked at the door for both, as a StackValidationError naming the
+ * field. Applies whether or not an `id` is supplied: a clock field outside
+ * this range is unrepresentable regardless of where the ID came from.
+ */
+function validateClockField(value, path) {
+    if (value === undefined)
+        return [];
+    // Type-checked callers always pass a Date, but this option is now
+    // reachable from the wire: JSON has no Date, so a server that forwards a
+    // parsed `POST /records` body hands us the ISO *string* it deserialized.
+    // Without this the very next line is `"2020-…".getTime()` — an unhandled
+    // TypeError, a 500 where the caller should have got a 400 naming the
+    // field.
+    if (!(value instanceof Date)) {
+        return [{ path, message: `${path} must be a Date.` }];
+    }
+    const ms = value.getTime();
+    if (Number.isNaN(ms)) {
+        return [{ path, message: `${path} is not a valid Date.` }];
+    }
+    if (ms < 0 || ms > MAX_ID_TIMESTAMP) {
+        return [
+            {
+                path,
+                message: `${path} is outside the representable range ` +
+                    `(1970-01-01T00:00:00.000Z…${new Date(MAX_ID_TIMESTAMP).toISOString()}).`,
+            },
+        ];
+    }
+    return [];
+}
+/**
+ * Timestamp-prefix plausibility check, shared by callers that compare an
+ * ID's embedded millisecond against a different reference each:
+ * ScopedStack.create() against the current time for any non-backdated
+ * create (a grantee is untrusted and could otherwise mint an ID that
+ * forges its sort position, and this is the one check standing between a
+ * delegated or grantee caller and doing so), and Stack.create() — reached
+ * directly when unscoped, or via ScopedStack.create() when the requester
+ * is the owner acting alone with an explicit `createdAt` — against that
+ * `createdAt` instead (the two must agree, not silently diverge — see
+ * docs/spec/data-model.md § Record IDs). Pass null to disable.
  */
-function validateIdTimestampSkew(id, toleranceMs) {
+function validateIdTimestampSkew(id, toleranceMs, referenceMs, referenceLabel) {
     if (toleranceMs === null)
         return;
-    const skew = Math.abs(Date.now() - idTimestamp(id));
+    const skew = Math.abs(referenceMs - idTimestamp(id));
     if (skew > toleranceMs) {
         throw new StackValidationError([
             {
                 path: 'id',
-                message: `ID "${id}" timestamp is outside the allowed clock-skew tolerance (${toleranceMs}ms).`,
+                message: `ID "${id}" timestamp disagrees with ${referenceLabel} by more than the allowed clock-skew tolerance (${toleranceMs}ms).`,
             },
         ]);
     }
@@ -689,7 +746,7 @@ export class Stack {
      */
     async ensureOwnerEntity(profile) {
         const entityTypeId = `${SYSTEM_TYPES.ENTITY}@1`;
-        const existing = await findFirstMatch((q) => this.query(q), { filter: { baseId: SYSTEM_TYPES.ENTITY, includeDeleted: true } }, (r) => r.content.did === this.ownerEntityId);
+        const existing = await findFirstMatch((q) => this.query(q), { filter: { baseId: SYSTEM_TYPES.ENTITY, includeDeleted: true, includeUnlisted: true } }, (r) => r.content.did === this.ownerEntityId);
         if (existing)
             return;
         await this.create(entityTypeId, {
@@ -898,7 +955,7 @@ export class Stack {
             let cursor;
             do {
                 const result = await this.adapter.queryRecords({
-                    filter: { typeId, includeDeleted: true },
+                    filter: { typeId, includeDeleted: true, includeUnlisted: true },
                     limit: 100,
                     cursor,
                 });
@@ -922,7 +979,12 @@ export class Stack {
      * Create a new record. Validates content against the type's schema.
      * `_group` records get their author stamped as the first `admin` roster
      * association here — the single stamping site for both Stack.create()
-     * and ScopedStack.create(). See docs/spec/identity.md § Group.
+     * and ScopedStack.create(). `opts.createdAt`/`updatedAt` let a caller
+     * backdate an imported record's clock fields — unconditionally here;
+     * ScopedStack.create() forwards to this same method, but only reaches
+     * this far with them when the requester is the stack owner acting alone
+     * — see BackdatableCreateRecordOptions and docs/spec/data-model.md §
+     * Record IDs.
      */
     async create(typeId, content, opts = {}) {
         this.assertOpen();
@@ -930,12 +992,36 @@ export class Stack {
         if (!type) {
             throw new Error(`Unknown type: "${typeId}". Call defineType() first.`);
         }
+        // updatedAt defaults to createdAt, not to the actual current time, so a
+        // plain import doesn't fabricate a fake edit.
+        //
+        // Copied, never aliased: the caller keeps its own reference to any Date
+        // it passed, and an import loop that reuses one Date across rows
+        // (`d.setTime(...)` per record — the obvious way to write one) would
+        // otherwise retro-edit every record it had already written, with no
+        // version bump and no change event. createdAt drives updatedAt and
+        // unlistedAt below, so one copy taken here covers all three.
+        // `instanceof Date`, not `!== undefined`: this runs before the error
+        // block below, so a non-Date reaching here off the wire would throw a
+        // raw TypeError out of .getTime() before validateClockField() could
+        // report it. Falling back keeps this line total; the error it recorded
+        // still throws below, so the fallback value is never actually stored.
+        const createdAt = opts.createdAt instanceof Date ? new Date(opts.createdAt.getTime()) : new Date();
+        const updatedAt = opts.updatedAt instanceof Date ? new Date(opts.updatedAt.getTime()) : createdAt;
         const errors = [
             ...validateReservedKeys(content),
             ...validateContent(content, type.schema),
             ...validatePermissions(opts.permissions),
             ...validateAssociations(opts.associations),
+            ...validateClockField(opts.createdAt, 'createdAt'),
+            ...validateClockField(opts.updatedAt, 'updatedAt'),
         ];
+        // Compared against the *effective* createdAt, so an updatedAt supplied
+        // on its own is caught too: defaulted-createdAt is now, which a
+        // backdated updatedAt alone would still precede.
+        if (opts.updatedAt !== undefined && updatedAt.getTime() < createdAt.getTime()) {
+            errors.push({ path: 'updatedAt', message: 'updatedAt cannot precede createdAt.' });
+        }
         if (errors.length > 0) {
             throw new StackValidationError(errors);
         }
@@ -944,17 +1030,35 @@ export class Stack {
             await this.checkAttachmentMimeTypeOnCreate(content);
         }
         await this.checkBindingsOnCreate(typeId, content);
-        if (opts.id !== undefined)
+        if (opts.id !== undefined) {
             validateRecordId(opts.id);
+            // Only when both are explicit: an `id` alone (no createdAt) stays a
+            // pure position choice, exactly as before this option existed.
+            if (opts.createdAt !== undefined) {
+                validateIdTimestampSkew(opts.id, this.idTimestampSkewMsValue, opts.createdAt.getTime(), 'createdAt');
+            }
+        }
         const associations = baseIdOf(typeId) === SYSTEM_TYPES.GROUP
             ? stampGroupAdmin(opts.associations, opts.entityId ?? this.ownerEntityId)
             : opts.associations;
-        const now = new Date();
+        // createdAt (hoisted above, alongside its validation) drives the ID
+        // when the caller doesn't supply one, so the two agree by construction
+        // rather than by coincidence — the same relationship an explicit `id`
+        // is checked against above.
+        // An explicit createdAt mints via generateIdForTimestamp(), which never
+        // clamps to "now" — generateId()'s monotonic floor would otherwise
+        // silently pull a deliberately historical id forward once this process
+        // has minted any live id past it. The no-createdAt path keeps
+        // generateId(), unaffected and still monotonic-safe.
+        const id = opts.id ??
+            (opts.createdAt !== undefined
+                ? generateIdForTimestamp(createdAt.getTime())
+                : generateId(createdAt.getTime()));
         const record = {
-            id: opts.id ?? generateId(),
+            id,
             typeId,
-            createdAt: now,
-            updatedAt: now,
+            createdAt,
+            updatedAt,
             content,
             version: 1,
             ...(opts.parentId && { parentId: opts.parentId }),
@@ -968,6 +1072,7 @@ export class Stack {
             ...(opts.principalId && { updatedVia: opts.principalId }),
             ...(opts.permissions?.length && { permissions: opts.permissions }),
             ...(associations?.length && { associations }),
+            ...(opts.unlisted && { unlistedAt: createdAt }),
         };
         const created = await this.adapter.createRecord(record);
         this.emitChange('create', created);
@@ -1139,6 +1244,35 @@ export class Stack {
         });
         this.emitChange('permissions', updated);
     }
+    /**
+     * Withhold a record from enumeration, or restore it. Orthogonal to
+     * setPermissions(): it says nothing about who may read the record, only
+     * whether `query()` and the change feed enumerate it by default. No-op if
+     * already in the requested state. See docs/spec/access-control.md §
+     * Unlisted records.
+     *
+     * The op passed to emitChange() carries the transition direction —
+     * `unlist` (kind `deleted`, so subscribers already holding the record are
+     * told to drop it) or `list` (kind `changed`, an upsert like `undelete`,
+     * for the record's publish moment).
+     */
+    async setUnlisted(id, unlisted, opts = {}) {
+        this.assertOpen();
+        const existing = await this.adapter.getRecord(id);
+        if (!existing) {
+            throw new StackNotFoundError(`Record not found: "${id}"`);
+        }
+        this.checkIfVersion(existing, opts.ifVersion);
+        if (Boolean(existing.unlistedAt) === unlisted)
+            return;
+        const updated = await this.adapter.setUnlisted(id, unlisted, {
+            expectedVersion: opts.ifVersion,
+            snapshot: this.buildVersionSnapshot(existing),
+            updatedBy: opts.updatedBy,
+            updatedVia: opts.updatedVia,
+        });
+        this.emitChange(unlisted ? 'unlist' : 'list', updated);
+    }
     /**
      * Soft-delete a record (default) or hard-delete it permanently, removing
      * the record and its version history. Soft delete snapshots and bumps
@@ -1477,6 +1611,7 @@ export class Stack {
             filter: {
                 baseId: family,
                 includeDeleted: true,
+                includeUnlisted: true,
                 ...(this.features.contentFieldQuery && { content: { [field]: value } }),
             },
         }, (r) => r.id !== excludeId && r.content[field] === value);
@@ -1531,6 +1666,7 @@ export class Stack {
             filter: {
                 typeId: metadataTypeId,
                 includeDeleted: true,
+                includeUnlisted: true,
                 ...(this.features.contentFieldQuery && { content: { fileId } }),
             },
         });
@@ -1688,22 +1824,24 @@ export class Stack {
      * race between the reference check below and the deletes it guards.
      */
     async deleteUnreferencedAttachmentRecordsFallback(fileId, metadataTypeId, opts = {}) {
-        // A soft-deleted record still counts as a reference — it must find its
-        // attachments intact on undelete. See docs/spec/attachments.md
-        // § Deleting attachments.
+        // A soft-deleted or unlisted record still counts as a reference — it
+        // must find its attachments intact on undelete or relisting. See
+        // docs/spec/attachments.md § Deleting attachments.
         const refResult = await this.query({
-            filter: { attachmentFileId: fileId, includeDeleted: true },
+            filter: { attachmentFileId: fileId, includeDeleted: true, includeUnlisted: true },
             limit: 1,
         });
         if (refResult.records.length > 0) {
             throw new StackConflictError('Attachment is still referenced by one or more records');
         }
-        // Cursor-walk with includeDeleted: metadata past page one, or soft-
-        // deleted, must be cleaned up too — not left pointing at deleted bytes.
+        // Cursor-walk with includeDeleted/includeUnlisted: metadata past page
+        // one, soft-deleted, or unlisted must be cleaned up too — not left
+        // pointing at deleted bytes.
         const metaResults = await queryAllPages((q) => this.query(q), {
             filter: {
                 typeId: metadataTypeId,
                 includeDeleted: true,
+                includeUnlisted: true,
                 ...(this.features.contentFieldQuery && { content: { fileId } }),
             },
         });
@@ -1728,7 +1866,7 @@ export class Stack {
         const now = Date.now();
         const metadataTypeId = `${SYSTEM_TYPES.ATTACHMENT}@1`;
         const metaRecords = await queryAllPages((q) => this.query(q), {
-            filter: { typeId: metadataTypeId, includeDeleted: true },
+            filter: { typeId: metadataTypeId, includeDeleted: true, includeUnlisted: true },
         });
         // Newest metadata record's createdAt per fileId, and its size (constant
         // across records sharing a fileId, since content-addressing guarantees
@@ -1755,7 +1893,7 @@ export class Stack {
         let reclaimedBytes = 0;
         for (const fileId of candidateFileIds) {
             const refResult = await this.query({
-                filter: { attachmentFileId: fileId, includeDeleted: true },
+                filter: { attachmentFileId: fileId, includeDeleted: true, includeUnlisted: true },
                 limit: 1,
             });
             if (refResult.records.length > 0)
@@ -1841,6 +1979,7 @@ export class Stack {
         const stop = await this.adapter.subscribeChanges({
             ...(opts.filter !== undefined && { filter: opts.filter }),
             ...(opts.includeRecords !== undefined && { includeRecords: opts.includeRecords }),
+            ...(opts.includeUnlisted !== undefined && { includeUnlisted: opts.includeUnlisted }),
             ...(opts.onError !== undefined && { onError: opts.onError }),
             ...(opts.onReset !== undefined && { onReset: opts.onReset }),
         }, (change) => delivery.deliver(change));
@@ -2298,6 +2437,8 @@ class ScopedSubscription extends Subscription {
     async filterAndDeliver(emission) {
         if (this.isClosed)
             return;
+        if (!passesUnlistedBoundary(emission, this.opts.includeUnlisted))
+            return;
         try {
             if (!(await this.canRead(emission.record, this.cache)))
                 return;
@@ -2716,12 +2857,29 @@ export class ScopedStack {
      * reference-creating options gated, and non-owner `_attachment@1`
      * creation refused save one carve-out. A scoped create always stamps
      * authorship — an absent entityId means an unscoped `Stack` wrote it.
-     * See docs/spec/access-control.md and docs/spec/attachments.md.
+     * `createdAt`/`updatedAt` are refused to everyone but the owner acting
+     * alone — see the guard below and docs/spec/data-model.md § Record IDs.
+     * See also docs/spec/access-control.md and docs/spec/attachments.md.
      */
     async create(typeId, content, opts = {}) {
         const principal = this.principalEntityId;
         if (!principal)
             throw new StackPermissionError('Anonymous requesters cannot create records');
+        // createdAt/updatedAt let a caller backdate a record's clock fields —
+        // and, without `id` also supplied, its sort position too. Refused to
+        // everyone but the owner acting alone (undelegated, authenticated as
+        // themselves): a grantee is exactly the untrusted actor the `id`
+        // skew check below already exists to stop from forging a sort
+        // position, and a delegated app acting for the owner inherits none of
+        // the owner's extra trust — same reasoning as mayGrantAccess() below.
+        // Refused rather than silently dropped, so an app never believes it
+        // published something it didn't. This is also the enforcement a
+        // server built on ScopedStack inherits for `POST /records`: an
+        // owner-authenticated request may carry both fields, anyone else's
+        // has them ignored.
+        if (('createdAt' in opts || 'updatedAt' in opts) && !this.ownerActingAlone) {
+            throw new StackPermissionError('createdAt/updatedAt can only be set by the stack owner acting alone; a grantee or delegated create always stamps the current time.');
+        }
         if (!(await this.checkCreateGrant(typeId))) {
             throw new StackPermissionError(`No create grant for type "${typeId}"`);
         }
@@ -2738,11 +2896,22 @@ export class ScopedStack {
         if (opts.permissions?.length && !this.mayGrantAccess()) {
             throw new StackPermissionError('A delegated principal cannot set permissions, at create time or after');
         }
+        if (opts.unlisted && !this.mayGrantAccess()) {
+            throw new StackPermissionError('A delegated principal cannot create an unlisted record, at create time or after');
+        }
         this.requireOwnerForOwnerDid(typeId, content.did);
         await this.requireAppIdMatchesPrincipal(opts.appId);
         if (opts.id !== undefined) {
             validateRecordId(opts.id);
-            validateIdTimestampSkew(opts.id, this.idTimestampSkewMs);
+            // Skipped when createdAt is also supplied: only the owner reaches
+            // here with that combination (checked above), and Stack.create()
+            // below checks the id against createdAt instead of "now" — the
+            // check here exists for a live grantee write, and a backdated
+            // owner create is deliberately not one. See
+            // docs/spec/data-model.md § Record IDs.
+            if (opts.createdAt === undefined) {
+                validateIdTimestampSkew(opts.id, this.idTimestampSkewMs, Date.now(), 'the current time');
+            }
         }
         if (opts.parentId !== undefined && !(await this.canReadReferent(opts.parentId))) {
             throw new StackPermissionError();
@@ -2803,6 +2972,9 @@ export class ScopedStack {
     async query(query = {}) {
         assertValidSort(query.sort);
         assertValidRelatedTo(query.filter?.relatedTo);
+        if (query.filter?.includeUnlisted && !this.ownerActingAlone) {
+            throw new StackPermissionError('includeUnlisted is owner-only');
+        }
         const limit = Math.min(query.limit ?? DEFAULT_QUERY_LIMIT, MAX_QUERY_LIMIT);
         const records = [];
         const maxFetched = limit * 10;
@@ -2888,6 +3060,7 @@ export class ScopedStack {
             filter: {
                 baseId: SYSTEM_TYPES.APP,
                 includeDeleted: true,
+                includeUnlisted: true,
                 ...(this.stack.features.contentFieldQuery && {
                     content: { did: this.principalEntityId },
                 }),
@@ -2964,6 +3137,30 @@ export class ScopedStack {
         }
         return this.stack.setPermissions(id, permissions, { ...opts, ...this.actor });
     }
+    /**
+     * Withhold a record from enumeration, or restore it — gated exactly like
+     * setPermissions(), since both decide who or what can discover the
+     * record rather than merely read it once found. See
+     * docs/spec/access-control.md § Unlisted records.
+     */
+    async setUnlisted(id, unlisted, opts = {}) {
+        const record = await this.stack.get(id);
+        if (!record)
+            throw new StackNotFoundError(`Record not found: "${id}"`);
+        await this.requireOwnerForGrantRecord(record);
+        if (baseIdOf(record.typeId) === SYSTEM_TYPES.GROUP) {
+            if (!this.isGroupManager(record))
+                throw await this.denialFor(record);
+        }
+        else {
+            if (!this.mayReshare(this.principalEntityId, record))
+                throw await this.denialFor(record);
+            if (this.delegated && !this.mayReshare(this.subjectEntityId, record)) {
+                throw await this.denialFor(record);
+            }
+        }
+        return this.stack.setUnlisted(id, unlisted, { ...opts, ...this.actor });
+    }
     /**
      * Hard delete is owner-only: it is irreversible and destroys version
      * history, so neither the write bit nor delete-own/delete-any grants
@@ -3145,6 +3342,9 @@ export class ScopedStack {
                 'a relayed frame was already scoped by the session that opened it, and a purge leaves ' +
                 'no record to re-check. Subscribe with a session-scoped stack instead.');
         }
+        if (opts.includeUnlisted && !this.ownerActingAlone) {
+            throw new StackPermissionError('includeUnlisted is owner-only');
+        }
         return this.changes.add(new ScopedSubscription((record, cache) => this.canReadCached(record, cache), handler, opts));
     }
     /**