@noodleseed/one 0.162.0 → 0.163.0

package/node_modules/@noodle-borg/service/dist/business-information/source-ingestion-coordinator.js

@@ -0,0 +1,136 @@
+import { sourceErrorCode } from './source-ingestion-failures.js';
+import { validateManagedPayload } from './validation.js';
+/** Executes only the normalized, read-only source protocol and commits through a fenced store. */
+export class SourceIngestionCoordinator {
+    #store;
+    #executor;
+    #workerId;
+    #validateRecord;
+    #now;
+    #leaseMs;
+    #retryMs;
+    #pageLimit;
+    #maxPagesPerRun;
+    constructor(options) {
+        this.#store = options.store;
+        this.#executor = options.executor;
+        this.#workerId = boundedToken('source worker id', options.workerId, 128);
+        this.#validateRecord = options.validateRecord;
+        this.#now = options.now ?? (() => new Date());
+        this.#leaseMs = boundedInteger('source lease', options.leaseMs ?? 60_000, 1_000, 15 * 60_000);
+        this.#retryMs = boundedInteger('source retry', options.retryMs ?? 30_000, 1_000, 86_400_000);
+        this.#pageLimit = boundedInteger('source page limit', options.pageLimit ?? 100, 1, 100);
+        this.#maxPagesPerRun = boundedInteger('source maximum pages', options.maxPagesPerRun ?? 100, 1, 10_000);
+    }
+    async runOne() {
+        let lease = await this.#store.claimDue({
+            now: this.#now(),
+            workerId: this.#workerId,
+            leaseMs: this.#leaseMs,
+        });
+        if (lease === undefined)
+            return { disposition: 'idle' };
+        let pages = 0;
+        try {
+            while (pages < this.#maxPagesPerRun) {
+                const sourcePage = await this.#executor.scan({
+                    binding: lease.binding,
+                    operation: lease.binding.scan,
+                    request: requestFor(lease, this.#pageLimit),
+                });
+                if (sourcePage.resetRequired === true) {
+                    validateResetPage(sourcePage);
+                    await this.#store.resetCheckpoint({
+                        lease,
+                        now: this.#now(),
+                        errorCode: 'checkpoint_reset_required',
+                    });
+                    return { disposition: 'reset', pages };
+                }
+                const page = normalizePage(lease.binding, sourcePage, this.#validateRecord);
+                const committed = await this.#store.commitPage({ lease, now: this.#now(), page });
+                if (!committed.ok)
+                    return { disposition: 'lost_lease', pages };
+                pages += 1;
+                if (page.complete) {
+                    return { disposition: 'completed', binding: committed.binding, pages };
+                }
+                if (committed.lease === undefined)
+                    throw new Error('source page commit lost its continuation lease');
+                lease = committed.lease;
+            }
+            throw sourceError('source_page_limit_exceeded');
+        }
+        catch (error) {
+            if (sourceErrorCode(error) === 'invalid_checkpoint') {
+                await this.#store.resetCheckpoint({
+                    lease,
+                    now: this.#now(),
+                    errorCode: 'checkpoint_reset_required',
+                });
+                return { disposition: 'reset', pages };
+            }
+            const now = this.#now();
+            await this.#store.failLease({
+                lease,
+                now,
+                errorCode: sourceErrorCode(error),
+                retryAt: new Date(now.getTime() + this.#retryMs),
+            });
+            throw error;
+        }
+    }
+}
+function requestFor(lease, limit) {
+    return {
+        mode: lease.mode,
+        ...(lease.cursor === undefined ? {} : { cursor: lease.cursor }),
+        ...(lease.mode === 'changes' && lease.checkpoint !== undefined
+            ? { checkpoint: lease.checkpoint }
+            : {}),
+        limit,
+    };
+}
+function normalizePage(binding, page, validateRecord) {
+    if (page.records.length > 100 || page.deletedIds.length > 100) {
+        throw sourceError('source_page_limit_exceeded');
+    }
+    const records = page.records.map((item) => ({
+        id: item.id,
+        ...(item.version === undefined ? {} : { version: item.version }),
+        record: validateManagedPayload(validateRecord(binding, item.record)),
+    }));
+    return {
+        records,
+        deletedIds: [...page.deletedIds],
+        ...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }),
+        ...(page.checkpoint === undefined ? {} : { checkpoint: page.checkpoint }),
+        complete: page.complete,
+    };
+}
+function validateResetPage(page) {
+    if (page.complete ||
+        page.records.length > 0 ||
+        page.deletedIds.length > 0 ||
+        page.nextCursor !== undefined ||
+        page.checkpoint !== undefined) {
+        throw sourceError('invalid_source_reset');
+    }
+}
+function sourceError(code) {
+    return Object.assign(new Error(code.replaceAll('_', ' ')), { code });
+}
+function boundedInteger(label, value, minimum, maximum) {
+    if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
+        throw new Error(`${label} is outside the supported range`);
+    }
+    return value;
+}
+function boundedToken(label, value, maximum) {
+    const normalized = value.trim();
+    if (normalized.length === 0 || normalized.length > maximum || /[\0\r\n]/.test(normalized)) {
+        throw new Error(`${label} is invalid`);
+    }
+    return normalized;
+}
+//# sourceMappingURL=source-ingestion-coordinator.js.map

package/node_modules/@noodle-borg/service/dist/business-information/source-ingestion-failures.js

@@ -0,0 +1,32 @@
+const AUTHORIZATION_LOSS_CODES = new Set([
+    'credential_unavailable',
+    'source_authorization_lost',
+    'authentication_required',
+    'authorization_required',
+    'unauthorized',
+    'forbidden',
+    'invalid_grant',
+    'invalid_token',
+    'access_denied',
+    'access_revoked',
+    'upstream_authentication_lost',
+]);
+export function normalizeSourceFailureCode(code) {
+    return AUTHORIZATION_LOSS_CODES.has(code) ? 'source_authorization_lost' : code;
+}
+export function sourceFailureHealth(errorCode) {
+    return normalizeSourceFailureCode(errorCode) === 'source_authorization_lost'
+        ? 'reauth_required'
+        : 'failed';
+}
+export function sourceErrorCode(error) {
+    if (typeof error === 'object' &&
+        error !== null &&
+        'code' in error &&
+        typeof error.code === 'string' &&
+        /^[a-z0-9_]{1,80}$/.test(error.code)) {
+        return normalizeSourceFailureCode(error.code);
+    }
+    return 'source_scan_failed';
+}
+//# sourceMappingURL=source-ingestion-failures.js.map

package/node_modules/@noodle-borg/service/dist/business-information/source-ingestion-memory-state.js

@@ -0,0 +1,154 @@
+import { createHash } from 'node:crypto';
+import { scopeKey } from './pagination.js';
+import { boundedInteger, clone, fixedDigest, jsonDigest, requiredCursor, requiredScanMode, sourceToken, suppressionReason, validInstant, } from './source-ingestion-memory-values.js';
+import { refreshReplayExpiresAt } from './source-refresh-retention.js';
+import { validateScalar, validateScope } from './validation.js';
+export function normalizeSuppression(value) {
+    return {
+        scope: clone(validateScope(value.scope)),
+        collectionKey: validateScalar('source collection key', value.collectionKey, 128),
+        id: validateScalar('source binding id', value.id, 128),
+        bindingGeneration: boundedInteger('source binding generation', value.bindingGeneration, 1, Number.MAX_SAFE_INTEGER),
+        sourceIdentityDigest: fixedDigest('source identity digest', value.sourceIdentityDigest),
+        reason: suppressionReason(value.reason),
+        erasedAt: validInstant('source suppression time', value.erasedAt),
+    };
+}
+export function validatePage(page) {
+    if (page.records.length > 100 || page.deletedIds.length > 100) {
+        throw new Error('source page exceeds the record limit');
+    }
+    if (page.resetRequired)
+        throw new Error('source reset page must not be committed');
+    if (page.complete && page.nextCursor !== undefined) {
+        throw new Error('complete source page cannot include a next cursor');
+    }
+    if (!page.complete && page.nextCursor === undefined) {
+        throw new Error('incomplete source page requires a next cursor');
+    }
+    const ids = new Set();
+    for (const item of page.records) {
+        const id = sourceToken('source record id', item.id);
+        if (ids.has(id))
+            throw new Error('source page contains duplicate record ids');
+        ids.add(id);
+    }
+    for (const raw of page.deletedIds) {
+        const id = sourceToken('deleted source id', raw);
+        if (ids.has(id))
+            throw new Error('source page records and deletions overlap');
+        ids.add(id);
+    }
+}
+export function completedBinding(current, completeAt, checkpoint) {
+    const { cursor: ignoredCursor, scanMode: ignoredMode, leaseOwner: ignoredOwner, leaseExpiresAt: ignoredExpiry, errorCode: ignoredError, checkpoint: ignoredCheckpoint, ...rest } = current;
+    void ignoredCursor;
+    void ignoredMode;
+    void ignoredOwner;
+    void ignoredExpiry;
+    void ignoredError;
+    void ignoredCheckpoint;
+    return {
+        ...rest,
+        ...(checkpoint === undefined
+            ? {}
+            : { checkpoint: sourceToken('source checkpoint', checkpoint) }),
+        health: 'current',
+        completeness: 'complete',
+        lastSuccessfulSyncAt: completeAt,
+        nextAttemptAt: new Date(Date.parse(completeAt) + current.pollIntervalMs).toISOString(),
+        revision: current.revision + 1,
+        updatedAt: completeAt,
+    };
+}
+export function withoutScanState(current, update) {
+    const { checkpoint: ignoredCheckpoint, cursor: ignoredCursor, scanMode: ignoredMode, leaseOwner: ignoredOwner, leaseExpiresAt: ignoredExpiry, ...rest } = current;
+    void ignoredCheckpoint;
+    void ignoredCursor;
+    void ignoredMode;
+    void ignoredOwner;
+    void ignoredExpiry;
+    return { ...rest, ...update };
+}
+export function leaseFrom(record, owner, leaseMs) {
+    return {
+        binding: clone(record),
+        owner,
+        fence: record.fence,
+        scanGeneration: record.scanGeneration,
+        mode: requiredScanMode(record.scanMode),
+        ...(record.cursor === undefined ? {} : { cursor: record.cursor }),
+        ...(record.checkpoint === undefined ? {} : { checkpoint: record.checkpoint }),
+        expiresAt: requiredCursor(record.leaseExpiresAt),
+        leaseMs,
+    };
+}
+export function ownsLease(current, lease, now, allowExpired = false) {
+    return (current !== undefined &&
+        current.state === 'active' &&
+        current.fence === lease.fence &&
+        current.scanGeneration === lease.scanGeneration &&
+        current.leaseOwner === lease.owner &&
+        (allowExpired || Date.parse(current.leaseExpiresAt ?? '') > now.getTime()));
+}
+export function due(record, now) {
+    return (record.state === 'active' &&
+        Date.parse(record.nextAttemptAt ?? record.createdAt) <= now.getTime() &&
+        (record.leaseExpiresAt === undefined || Date.parse(record.leaseExpiresAt) <= now.getTime()));
+}
+export function dueOrder(left, right) {
+    return ((left.nextAttemptAt ?? left.createdAt).localeCompare(right.nextAttemptAt ?? right.createdAt) ||
+        bindingKey(left).localeCompare(bindingKey(right)));
+}
+export function bindingKey(input) {
+    return `${scopeKey(validateScope(input.scope))}\0${validateScalar('source collection key', input.collectionKey, 128)}\0${validateScalar('source binding id', input.id, 128)}`;
+}
+export function externalRecordKey(binding, digest) {
+    return `${bindingKey(binding)}\0record\0${digest}`;
+}
+export function suppressionKey(binding, digest) {
+    return `${bindingKey(binding)}\0suppression\0${digest}`;
+}
+export function bindingFingerprint(value) {
+    return jsonDigest(value);
+}
+export function refreshIdempotencyDigest(value) {
+    return createHash('sha256')
+        .update(validateScalar('source refresh idempotency key', value, 256))
+        .digest('hex');
+}
+export function refreshRequestKey(key, generation, digest) {
+    return `${key}\0refresh\0${generation}\0${digest}`;
+}
+export function refreshJobId(key, generation, digest, scanGeneration) {
+    return `sync_${createHash('sha256')
+        .update(key)
+        .update('\0')
+        .update(String(generation))
+        .update('\0')
+        .update(digest)
+        .update('\0')
+        .update(String(scanGeneration))
+        .digest('hex')
+        .slice(0, 24)}`;
+}
+export function refreshReceipt(request, coalesced) {
+    return {
+        id: request.jobId,
+        state: request.state,
+        coalesced,
+        requestedAt: request.requestedAt,
+        ...(request.terminalAt === undefined
+            ? {}
+            : { replayExpiresAt: refreshReplayExpiresAt(request.terminalAt) }),
+    };
+}
+export function externalRecordId(binding, sourceIdentityDigest) {
+    return `ext_${createHash('sha256')
+        .update(sourceIdentityDigest)
+        .update('\0')
+        .update(String(binding.generation))
+        .digest('hex')
+        .slice(0, 32)}`;
+}
+//# sourceMappingURL=source-ingestion-memory-state.js.map