@form-engine-ts/core 2.5.0 → 2.6.0

package/README.md

@@ -76,14 +76,24 @@ Results are ordered by `submittedAt`, then submission ID. Both boundaries are in
 
 ## Versioning, incremental analytics, and paged storage
 
-`cloneVersionToDraft`, `publishDraft`, and `deleteDraft` implement revision-checked version transitions as pure functions.
+`cloneVersionToDraft`, asynchronous `publishDraft`, and `deleteDraft` implement revision-checked version transitions as
+pure functions.
+Clone/delete operations accept `expectedRevision`; cloning rejects non-published sources, publish validation failures are
+returned as typed `validation_failed` issues, and successful publishing archives only a supplied actual published record,
+preserving its schema and metadata. `createPublishTransitionPlan` returns complete records plus expected/next revisions for
+storage adapters implementing `VersionedFormStorageAdapter` to commit atomically.
 `createResponseAccumulator` incrementally counts choices, answered/unanswered values, and numeric summaries without retaining
-free-text bodies. Independent accumulators for the same schema can be merged, and `finalize()` matches `aggregateResponses`.
+free-text bodies. In lenient mode, mismatched responses are skipped and exposed by `addMany()` and `getReport()` instead of
+being included silently. Independent accumulators for the same schema can be merged, and `finalize()` matches
+`aggregateResponses`.
 
 `exportResponsesToCsvStream` accepts an `AsyncIterable`, emits the BOM/header and one chunk per response, and supports
-custom `CsvColumnDef` columns. Formula-injection neutralization applies to both default and custom columns.
+custom `CsvColumnDef` columns. Custom getters may be asynchronous and receive the submission, form version, and schema. Use
+`pipeResponsesToCsvStream` to write to a Web `WritableStream` or Node-compatible writable while honoring backpressure.
+Formula-injection neutralization applies to both default and custom columns.
 
 Adapters implementing `PagedSubmissionStorageAdapter` expose `listSubmissionPage(formId, options)`. The opaque Base64
-cursor combines `submittedAt` and response ID, so equal timestamps do not produce gaps or duplicates.
+cursor combines `submittedAt` and response ID, so equal timestamps do not produce gaps or duplicates. `metadataFilters`
+and `filter` are applied before page sizing.
 
 See the [project documentation](https://github.com/nitta-a/form-engine-ts#readme) for the complete schema and API guide.

package/dist/index.cjs

@@ -30,6 +30,7 @@ __export(index_exports, {
   calculatePageVisibility: () => calculatePageVisibility,
   cloneVersionToDraft: () => cloneVersionToDraft,
   collectSchemaLocales: () => collectSchemaLocales,
+  createPublishTransitionPlan: () => createPublishTransitionPlan,
   createResponseAccumulator: () => createResponseAccumulator,
   createSubmission: () => createSubmission,
   decodeSubmissionCursor: () => decodeSubmissionCursor,
@@ -41,7 +42,9 @@ __export(index_exports, {
   exportResponsesToCsvStream: () => exportResponsesToCsvStream,
   isDisplayConditionSatisfied: () => isDisplayConditionSatisfied,
   isQuestionVisible: () => isQuestionVisible,
+  matchesSubmissionPageFilters: () => matchesSubmissionPageFilters,
   normalizeSubmissionPageSize: () => normalizeSubmissionPageSize,
+  pipeResponsesToCsvStream: () => pipeResponsesToCsvStream,
   populateSchemaTranslations: () => populateSchemaTranslations,
   publishDraft: () => publishDraft,
   resolveFormTranslation: () => resolveFormTranslation,
@@ -1101,17 +1104,38 @@ function aggregateResponses(schema, submissions) {
   };
 }
 function responseValues(submission) {
-  return "values" in submission ? submission.values : submission.answers;
+  if (typeof submission !== "object" || submission === null) return void 0;
+  if ("values" in submission) return submission.values;
+  return "answers" in submission ? submission.answers : void 0;
 }
 function responseIdentifier(submission) {
-  return "id" in submission ? submission.id : submission.responseId;
+  if (typeof submission !== "object" || submission === null) return "<unknown>";
+  if ("id" in submission && typeof submission.id === "string" && submission.id.length > 0) return submission.id;
+  if ("responseId" in submission && typeof submission.responseId === "string" && submission.responseId.length > 0) {
+    return submission.responseId;
+  }
+  return "<unknown>";
+}
+function isAnswerRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
 }
-function responseMismatch(schema, submission) {
+function responseProblem(schema, submission) {
+  if (typeof submission !== "object" || submission === null) {
+    return { reason: "invalid_structure", error: "Submission structure is invalid." };
+  }
+  const identifier = responseIdentifier(submission);
+  const values = responseValues(submission);
+  if (typeof identifier !== "string" || identifier.length === 0 || typeof submission.formId !== "string" || typeof submission.submittedAt !== "string" || !isAnswerRecord(values)) {
+    return { reason: "invalid_structure", error: `Submission ${String(identifier)} structure is invalid.` };
+  }
   if (submission.formId !== schema.id) {
-    return `Submission ${responseIdentifier(submission)} does not match form ${schema.id}.`;
+    return { reason: "form_id_mismatch", error: `Submission ${identifier} does not match form ${schema.id}.` };
   }
-  if ("formVersion" in submission && submission.formVersion !== schema.version) {
-    return `Submission ${responseIdentifier(submission)} does not match ${schema.id}@${schema.version}.`;
+  if ("formVersion" in submission && submission.formVersion !== void 0 && submission.formVersion !== schema.version) {
+    return {
+      reason: "version_mismatch",
+      error: `Submission ${identifier} does not match ${schema.id}@${schema.version}.`
+    };
   }
   return void 0;
 }
@@ -1119,6 +1143,7 @@ var IncrementalResponseAccumulator = class _IncrementalResponseAccumulator {
   #schema;
   #mode;
   #fields;
+  #skipReasons = [];
   #submissionCount = 0;
   constructor(schema, options) {
     assertValidFormSchema(schema);
@@ -1140,9 +1165,14 @@ var IncrementalResponseAccumulator = class _IncrementalResponseAccumulator {
     );
   }
   add(submission) {
-    const mismatch = responseMismatch(this.#schema, submission);
-    if (mismatch !== void 0 && this.#mode === "strict") return { success: false, error: mismatch };
+    const problem = responseProblem(this.#schema, submission);
+    if (problem !== void 0) {
+      if (this.#mode === "strict") return { success: false, error: problem.error };
+      this.#skipReasons.push({ responseId: responseIdentifier(submission), reason: problem.reason });
+      return { success: true, skipped: true };
+    }
     const values = responseValues(submission);
+    if (!isAnswerRecord(values)) throw new Error("Validated response answers are unavailable.");
     const visibility = calculateFieldVisibility(this.#schema, values);
     for (const field of this.#schema.fields) {
       const accumulator = this.#fields.get(field.id);
@@ -1172,6 +1202,7 @@ var IncrementalResponseAccumulator = class _IncrementalResponseAccumulator {
       const result = this.add(submission);
       if (!result.success) throw new TypeError(result.error ?? "Submission could not be accumulated.");
     }
+    return this.getReport();
   }
   merge(other) {
     if (!(other instanceof _IncrementalResponseAccumulator)) {
@@ -1181,6 +1212,7 @@ var IncrementalResponseAccumulator = class _IncrementalResponseAccumulator {
       throw new TypeError("Response accumulators must use the same schema.");
     }
     this.#submissionCount += other.#submissionCount;
+    this.#skipReasons.push(...other.#skipReasons);
     for (const [fieldId, source] of other.#fields) {
       const target = this.#fields.get(fieldId);
       if (target === void 0) throw new Error(`Accumulator for ${fieldId} is unavailable.`);
@@ -1196,6 +1228,13 @@ var IncrementalResponseAccumulator = class _IncrementalResponseAccumulator {
     }
     return this;
   }
+  getReport() {
+    return {
+      processedCount: this.#submissionCount,
+      skippedCount: this.#skipReasons.length,
+      skipReasons: this.#skipReasons.map((reason) => ({ ...reason }))
+    };
+  }
   finalize() {
     return {
       formId: this.#schema.id,
@@ -1267,6 +1306,7 @@ function asFormResponse(submission) {
     responseId: submission.id,
     formId: submission.formId,
     sourceLocale: submission.locale,
+    formVersion: submission.formVersion,
     answers: submission.values,
     submittedAt: submission.submittedAt,
     ...submission.metadata === void 0 ? {} : { metadata: submission.metadata },
@@ -1290,8 +1330,8 @@ async function* exportResponsesToCsvStream(schema, submissions, options = {}) {
   const header = headers.map((value) => escapeCsvCell(value, neutralizeFormulas)).join(",");
   yield `${options.withBom ?? true ? "\uFEFF" : ""}${header}`;
   for await (const submission of submissions) {
-    const mismatch = responseMismatch(schema, submission);
-    if (mismatch !== void 0) throw new TypeError(mismatch);
+    const problem = responseProblem(schema, submission);
+    if (problem !== void 0) throw new TypeError(problem.error);
     const response = asFormResponse(submission);
     const answers = response.answers;
     const visible = selectVisibleAnswers(schema, answers);
@@ -1301,11 +1341,64 @@ async function* exportResponsesToCsvStream(schema, submissions, options = {}) {
       response.sourceLocale ?? "",
       ...schema.fields.map((field) => serializeUnknown(visible[field.id]))
     ] : [];
-    const customCells = customColumns.map((column) => column.getValue(response));
+    const context = {
+      ...response,
+      submission: response,
+      formVersion: response.formVersion ?? schema.version,
+      schema
+    };
+    const customCells = await Promise.all(customColumns.map((column) => column.getValue(context)));
     yield `\r
 ${[...defaultCells, ...customCells].map((value) => escapeCsvCell(value, neutralizeFormulas)).join(",")}`;
   }
 }
+function isWebWritableStream(writable) {
+  return "getWriter" in writable && typeof writable.getWriter === "function";
+}
+async function writeNodeChunk(writable, chunk, streamError) {
+  if (writable.write(chunk)) return;
+  let onDrain;
+  const drain = new Promise((resolve) => {
+    onDrain = resolve;
+    writable.once("drain", resolve);
+  });
+  try {
+    await Promise.race([drain, streamError]);
+  } finally {
+    if (onDrain !== void 0) writable.removeListener("drain", onDrain);
+  }
+}
+async function pipeResponsesToCsvStream(schema, submissions, writable, options = {}) {
+  const encoder = new TextEncoder();
+  if (isWebWritableStream(writable)) {
+    const writer = writable.getWriter();
+    try {
+      for await (const chunk of exportResponsesToCsvStream(schema, submissions, options)) {
+        await writer.write(encoder.encode(chunk));
+      }
+      await writer.close();
+    } catch (cause) {
+      await writer.abort(cause);
+      throw cause;
+    } finally {
+      writer.releaseLock();
+    }
+    return;
+  }
+  let onStreamError;
+  const streamError = new Promise((_resolve, reject) => {
+    onStreamError = reject;
+    writable.once("error", reject);
+  });
+  try {
+    for await (const chunk of exportResponsesToCsvStream(schema, submissions, options)) {
+      await writeNodeChunk(writable, encoder.encode(chunk), streamError);
+    }
+    await Promise.race([new Promise((resolve) => writable.end(resolve)), streamError]);
+  } finally {
+    if (onStreamError !== void 0) writable.removeListener("error", onStreamError);
+  }
+}
 function exportResponsesToCsv(schema, responses, options = {}) {
   assertValidFormSchema(schema);
   for (const response of responses) {
@@ -1477,6 +1570,32 @@ function normalizeSubmissionPageSize(pageSize, fallback = 100) {
   if (!Number.isSafeInteger(value) || value < 1) throw new TypeError("pageSize must be a positive safe integer.");
   return value;
 }
+function isJsonArray(value) {
+  return Array.isArray(value);
+}
+function isJsonObject(value) {
+  return typeof value === "object" && value !== null && !isJsonArray(value);
+}
+function jsonValuesEqual(left, right) {
+  if (left === right) return true;
+  if (left === void 0 || left === null || right === void 0 || right === null || typeof left !== typeof right) {
+    return false;
+  }
+  if (isJsonArray(left) || isJsonArray(right)) {
+    return isJsonArray(left) && isJsonArray(right) && left.length === right.length && left.every((value, index) => jsonValuesEqual(value, right[index]));
+  }
+  if (!isJsonObject(left) || !isJsonObject(right)) return false;
+  const leftKeys = Object.keys(left);
+  const rightKeys = Object.keys(right);
+  return leftKeys.length === rightKeys.length && leftKeys.every((key) => Object.hasOwn(right, key) && jsonValuesEqual(left[key], right[key]));
+}
+function matchesSubmissionPageFilters(submission, options) {
+  if (options.filter !== void 0 && !options.filter(submission)) return false;
+  if (options.metadataFilters === void 0) return true;
+  return Object.entries(options.metadataFilters).every(
+    ([key, value]) => jsonValuesEqual(submission.metadata?.[key], value)
+  );
+}
 
 // src/validation.ts
 var DEFAULT_MESSAGES = {
@@ -1915,6 +2034,12 @@ async function resolveFormTranslation(schema, adapter, targetLocale, sourceLocal
 }
 
 // src/versioning.ts
+function revisionConflict(state, expectedRevision) {
+  return expectedRevision === void 0 || expectedRevision === state.revision ? void 0 : {
+    success: false,
+    error: { type: "revision_conflict", expectedRevision, actualRevision: state.revision }
+  };
+}
 function validateState(state) {
   if (state.formId.trim().length === 0) throw new TypeError("formId must not be empty.");
   if (!Number.isSafeInteger(state.nextVersion) || state.nextVersion < 1) {
@@ -1927,9 +2052,25 @@ function validateState(state) {
 function cloneVersionToDraft(state, sourceSchema, options = {}) {
   validateState(state);
   if (sourceSchema.id !== state.formId) throw new TypeError("sourceSchema.id must match state.formId.");
+  const conflict = revisionConflict(state, options.expectedRevision);
+  if (conflict !== void 0) return conflict;
   if (state.draftVersion !== void 0) {
     return { success: false, error: { type: "draft_already_exists", currentDraftVersion: state.draftVersion } };
   }
+  const allowedSourceVersions = /* @__PURE__ */ new Set([
+    ...state.publishedVersion === void 0 ? [] : [state.publishedVersion],
+    ...options.allowedSourceVersions ?? []
+  ]);
+  if (!allowedSourceVersions.has(sourceSchema.version)) {
+    return {
+      success: false,
+      error: {
+        type: "invalid_source_version",
+        requestedVersion: sourceSchema.version,
+        ...state.publishedVersion === void 0 ? {} : { publishedVersion: state.publishedVersion }
+      }
+    };
+  }
   const maxVersions = options.maxVersions ?? Number.MAX_SAFE_INTEGER;
   if (!Number.isSafeInteger(maxVersions) || maxVersions < 1) {
     throw new TypeError("maxVersions must be a positive safe integer.");
@@ -1951,25 +2092,42 @@ function cloneVersionToDraft(state, sourceSchema, options = {}) {
     }
   };
 }
-function publishDraft(state, draftSchema, options = {}) {
+function validatePublishedRecord(state, record) {
+  if (record === void 0) return;
+  if (record.formId !== state.formId || record.version !== state.publishedVersion || record.status !== "published" || record.schema.id !== record.formId || record.schema.version !== record.version) {
+    throw new TypeError("currentPublishedRecord must match the state's published version.");
+  }
+}
+function transitionTimestamp(options) {
+  const timestamp = options.publishedAt ?? options.timestamp ?? "1970-01-01T00:00:00.000Z";
+  if (!Number.isFinite(Date.parse(timestamp))) throw new TypeError("publishedAt must be a valid date string.");
+  return timestamp;
+}
+async function publishDraft(state, draftSchema, options = {}) {
   validateState(state);
-  if (options.expectedRevision !== void 0 && options.expectedRevision !== state.revision) {
+  const conflict = revisionConflict(state, options.expectedRevision);
+  if (conflict !== void 0) return conflict;
+  if (state.draftVersion === void 0 || draftSchema.id !== state.formId || draftSchema.version !== state.draftVersion) {
+    return { success: false, error: { type: "draft_not_found" } };
+  }
+  validatePublishedRecord(state, options.currentPublishedRecord);
+  const validation = await options.validate?.(draftSchema);
+  if (validation === false || Array.isArray(validation) && validation.length > 0) {
     return {
       success: false,
-      error: {
-        type: "revision_conflict",
-        expectedRevision: options.expectedRevision,
-        actualRevision: state.revision
-      }
+      error: { type: "validation_failed", issues: Array.isArray(validation) ? validation : [] }
     };
   }
-  if (state.draftVersion === void 0 || draftSchema.id !== state.formId || draftSchema.version !== state.draftVersion) {
-    return { success: false, error: { type: "draft_not_found" } };
-  }
-  if (options.validate?.(draftSchema) === false) throw new TypeError("Draft schema validation failed.");
-  const timestamp = options.timestamp ?? "1970-01-01T00:00:00.000Z";
-  if (!Number.isFinite(Date.parse(timestamp))) throw new TypeError("timestamp must be a valid date string.");
+  const timestamp = transitionTimestamp(options);
   const archivedVersion = state.publishedVersion;
+  const archivedRecords = options.currentPublishedRecord === void 0 ? [] : [
+    {
+      ...options.currentPublishedRecord,
+      status: "archived",
+      revision: options.currentPublishedRecord.revision + 1,
+      archivedAt: timestamp
+    }
+  ];
   const { draftVersion: _draftVersion, ...stateWithoutDraft } = state;
   return {
     success: true,
@@ -1984,15 +2142,57 @@ function publishDraft(state, draftSchema, options = {}) {
         version: draftSchema.version,
         status: "published",
         schema: draftSchema,
+        revision: 1,
+        ...archivedVersion === void 0 ? {} : { createdFromVersion: archivedVersion },
         createdAt: timestamp,
         publishedAt: timestamp
       },
+      archivedRecords,
       ...archivedVersion === void 0 ? {} : { archivedVersion }
     }
   };
 }
-function deleteDraft(state) {
+async function createPublishTransitionPlan(state, draftRecord, options = {}) {
+  if (draftRecord.formId !== state.formId || draftRecord.version !== state.draftVersion || draftRecord.schema.id !== draftRecord.formId || draftRecord.schema.version !== draftRecord.version) {
+    return { success: false, error: { type: "draft_not_found" } };
+  }
+  if (draftRecord.status !== "draft") {
+    return { success: false, error: { type: "version_immutable", status: draftRecord.status } };
+  }
+  const expectedRevision = options.expectedRevision ?? state.revision;
+  const timestamp = transitionTimestamp(options);
+  const result = await publishDraft(state, draftRecord.schema, {
+    ...options,
+    expectedRevision,
+    publishedAt: timestamp
+  });
+  if (!result.success) return result;
+  const publishedRecordToSave = {
+    ...draftRecord,
+    status: "published",
+    revision: draftRecord.revision + 1,
+    publishedAt: timestamp
+  };
+  return {
+    success: true,
+    value: {
+      nextState: result.value.nextState,
+      plan: {
+        formId: state.formId,
+        expectedRevision,
+        nextRevision: result.value.nextState.revision,
+        draftToDeleteVersion: draftRecord.version,
+        publishedRecordToSave,
+        archivedRecordsToSave: result.value.archivedRecords,
+        timestamp
+      }
+    }
+  };
+}
+function deleteDraft(state, options = {}) {
   validateState(state);
+  const conflict = revisionConflict(state, options.expectedRevision);
+  if (conflict !== void 0) return conflict;
   if (state.draftVersion === void 0) return { success: false, error: { type: "draft_not_found" } };
   const { draftVersion: _draftVersion, ...stateWithoutDraft } = state;
   return {
@@ -2018,6 +2218,7 @@ function assertVersionMutable(status) {
   calculatePageVisibility,
   cloneVersionToDraft,
   collectSchemaLocales,
+  createPublishTransitionPlan,
   createResponseAccumulator,
   createSubmission,
   decodeSubmissionCursor,
@@ -2029,7 +2230,9 @@ function assertVersionMutable(status) {
   exportResponsesToCsvStream,
   isDisplayConditionSatisfied,
   isQuestionVisible,
+  matchesSubmissionPageFilters,
   normalizeSubmissionPageSize,
+  pipeResponsesToCsvStream,
   populateSchemaTranslations,
   publishDraft,
   resolveFormTranslation,