@palantir/pack.document-schema.type-gen 0.9.0 → 0.11.0

package/build/esm/utils/schema/generateInternalFromSchema.js

@@ -15,109 +15,8 @@
  */
 
 import { GENERATED_FILE_HEADER } from "../generatedFileHeader.js";
-import { resolveSchemaChain } from "./resolveSchemaChain.js";
-import { findRecordExportName, isRecordSchema, isUnionSchema, TypeKind } from "./runtimeSchema.js";
-/**
- * Convert a schema field type to TypeScript string for internal types.
- */
-function convertTypeToTypeScript(fieldType) {
-  switch (fieldType.type) {
-    case TypeKind.ARRAY:
-      {
-        if (fieldType.items == null) {
-          throw new Error("Array field is missing items type");
-        }
-        const inner = convertTypeToTypeScript(fieldType.items);
-        // Wrap compound types (unions, other arrays) in parens before appending []
-        const needsParens = inner.includes("|") || inner.startsWith("readonly ");
-        const wrapped = needsParens ? `(${inner})` : inner;
-        return `readonly ${wrapped}[]`;
-      }
-    case TypeKind.BOOLEAN:
-      return "boolean";
-    case TypeKind.DOC_REF:
-      return "string";
-    case TypeKind.DOUBLE:
-      return "number";
-    case TypeKind.MEDIA_REF:
-      return "string";
-    case TypeKind.OBJECT_REF:
-      return "string";
-    case TypeKind.OPTIONAL:
-      if (fieldType.item == null) {
-        throw new Error("Optional field is missing inner type");
-      }
-      return `(${convertTypeToTypeScript(fieldType.item)} | undefined)`;
-    case TypeKind.REF:
-      return "unknown";
-    case TypeKind.STRING:
-      return "string";
-    case TypeKind.USER_REF:
-      return "string";
-    default:
-      throw new Error(`Unknown schema field type: ${fieldType.type}`);
-  }
-}
-
-/**
- * Convert a schema field type to a FieldTypeDescriptor source string.
- */
-function convertTypeToFieldTypeDescriptor(fieldType, schema) {
-  switch (fieldType.type) {
-    case TypeKind.ARRAY:
-      if (fieldType.items == null) {
-        throw new Error("Array field is missing items type");
-      }
-      return `{ kind: "array", element: ${convertTypeToFieldTypeDescriptor(fieldType.items, schema)} }`;
-    case TypeKind.OPTIONAL:
-      if (fieldType.item == null) {
-        throw new Error("Optional field is missing inner type");
-      }
-      return `{ kind: "optional", inner: ${convertTypeToFieldTypeDescriptor(fieldType.item, schema)} }`;
-    case TypeKind.REF:
-      {
-        if (fieldType.name == null) {
-          throw new Error("Ref field is missing a name");
-        }
-        const exportKey = fieldType.refType === "record" ? findRecordExportName(fieldType.name, schema) ?? fieldType.name : fieldType.name;
-        return `{ kind: "modelRef", model: "${exportKey}" }`;
-      }
-    default:
-      return `{ kind: "primitive" }`;
-  }
-}
-
-/**
- * Convert a schema field type to a Zod schema string.
- */
-function convertTypeToZodSchema(fieldType) {
-  switch (fieldType.type) {
-    case TypeKind.ARRAY:
-      if (fieldType.items == null) {
-        throw new Error("Array field is missing items type");
-      }
-      return `z.array(${convertTypeToZodSchema(fieldType.items)})`;
-    case TypeKind.BOOLEAN:
-      return "z.boolean()";
-    case TypeKind.DOC_REF:
-    case TypeKind.MEDIA_REF:
-    case TypeKind.OBJECT_REF:
-    case TypeKind.STRING:
-    case TypeKind.USER_REF:
-      return "z.string()";
-    case TypeKind.DOUBLE:
-      return "z.number()";
-    case TypeKind.OPTIONAL:
-      if (fieldType.item == null) {
-        throw new Error("Optional field is missing inner type");
-      }
-      return `${convertTypeToZodSchema(fieldType.item)}.optional()`;
-    case TypeKind.REF:
-      return "z.unknown()";
-    default:
-      throw new Error(`Unknown schema field type: ${fieldType.type}`);
-  }
-}
+import { convertFieldTypeToDescriptor, convertFieldTypeToInternalTypeScript, convertFieldTypeToInternalZod } from "../ir/irFieldHelpers.js";
+import { chainHasDerivedFields } from "./resolveSchemaChain.js";
 
 /** Collected info about all fields for a model across all versions. */
 
@@ -141,30 +40,35 @@ function collectRecordModels(chain) {
   // Initialize models from all versions
   for (const {
     version,
-    schema
+    ir
   } of chain) {
-    for (const [exportName, item] of Object.entries(schema)) {
-      if (!isRecordSchema(item)) continue;
-      if (!allRecordModels.has(exportName)) {
-        allRecordModels.set(exportName, {
+    for (const [modelKey, modelDef] of Object.entries(ir.models)) {
+      if (modelDef.type !== "record") continue;
+      if (!allRecordModels.has(modelKey)) {
+        allRecordModels.set(modelKey, {
           allFields: new Map(),
           steps: []
         });
       }
-      const model = allRecordModels.get(exportName);
-      for (const [fieldName, fieldType] of Object.entries(item.fields)) {
-        const existing = model.allFields.get(fieldName);
+      const model = allRecordModels.get(modelKey);
+      for (const field of modelDef.record.fields) {
+        const existing = model.allFields.get(field.key);
         if (existing == null) {
-          model.allFields.set(fieldName, {
-            fieldType,
+          model.allFields.set(field.key, {
+            fieldType: field.fieldType,
+            isOptional: field.isOptional === true,
             presentInVersions: new Set([version]),
-            alwaysOptional: fieldType.type === TypeKind.OPTIONAL
+            alwaysOptional: field.isOptional === true,
+            everOptional: field.isOptional === true
           });
         } else {
           existing.presentInVersions.add(version);
-          if (fieldType.type !== TypeKind.OPTIONAL) {
+          if (field.isOptional !== true) {
             existing.alwaysOptional = false;
           }
+          if (field.isOptional === true) {
+            existing.everOptional = true;
+          }
         }
       }
     }
@@ -172,32 +76,30 @@ function collectRecordModels(chain) {
 
   // Compute upgrade steps from version diffs
   for (let i = 1; i < chain.length; i++) {
-    const prevVersion = chain[i - 1];
-    const currVersion = chain[i];
-    for (const [exportName, currItem] of Object.entries(currVersion.schema)) {
-      if (!isRecordSchema(currItem)) continue;
-      const prevItem = prevVersion.schema[exportName];
-      if (prevItem == null || !isRecordSchema(prevItem)) continue;
-      const model = allRecordModels.get(exportName);
+    const prevEntry = chain[i - 1];
+    const currEntry = chain[i];
+    for (const [modelKey, currModelDef] of Object.entries(currEntry.ir.models)) {
+      if (currModelDef.type !== "record") continue;
+      const prevModelDef = prevEntry.ir.models[modelKey];
+      if (prevModelDef == null || prevModelDef.type !== "record") continue;
+      const model = allRecordModels.get(modelKey);
       if (model == null) continue;
-      const prevFields = new Set(Object.keys(prevItem.fields));
-      const currFields = new Set(Object.keys(currItem.fields));
+      const prevFields = new Set(prevModelDef.record.fields.map(f => f.key));
+      const currFields = new Set(currModelDef.record.fields.map(f => f.key));
 
       // Fields added in this version
       const addedFields = new Map();
-      const recordUpgrades = currVersion.migrations?.[exportName];
-      for (const fieldName of currFields) {
-        if (!prevFields.has(fieldName)) {
-          const annotation = recordUpgrades?.[fieldName];
+      const recordUpgrades = currEntry.migrations?.[modelKey];
+      for (const fieldKey of currFields) {
+        if (!prevFields.has(fieldKey)) {
+          const annotation = recordUpgrades?.[fieldKey];
           if (annotation != null) {
-            addedFields.set(fieldName, {
-              derivedFrom: [...annotation.derivedFrom],
-              forwardSource: annotation.forward.toString()
+            addedFields.set(fieldKey, {
+              derivedFrom: [...annotation.derivedFrom]
             });
           } else {
-            addedFields.set(fieldName, {
-              derivedFrom: [],
-              forwardSource: "() => undefined"
+            addedFields.set(fieldKey, {
+              derivedFrom: []
             });
           }
         }
@@ -205,15 +107,14 @@ function collectRecordModels(chain) {
 
       // Fields removed in this version
       const removedFields = [];
-      for (const fieldName of prevFields) {
-        if (!currFields.has(fieldName)) {
-          removedFields.push(fieldName);
+      for (const fieldKey of prevFields) {
+        if (!currFields.has(fieldKey)) {
+          removedFields.push(fieldKey);
         }
       }
       if (addedFields.size > 0 || removedFields.length > 0) {
         model.steps.push({
-          name: `v${currVersion.version}`,
-          addedInVersion: currVersion.version,
+          addedInVersion: currEntry.version,
           fields: addedFields,
           removedFields
         });
@@ -239,11 +140,11 @@ function collectRecordModels(chain) {
 function generateInternalTypes(allRecordModels) {
   let output = GENERATED_FILE_HEADER;
   output += "\n";
-  for (const [exportName, model] of sortedEntries(allRecordModels)) {
+  for (const [modelKey, model] of sortedEntries(allRecordModels)) {
     output += `/** Internal representation containing all fields across all schema versions. */\n`;
-    output += `export interface ${exportName}__Internal {\n`;
+    output += `export interface ${modelKey}__Internal {\n`;
     for (const [fieldName, fieldInfo] of sortedEntries(model.allFields)) {
-      const tsType = convertTypeToTypeScript(fieldInfo.fieldType);
+      const tsType = convertFieldTypeToInternalTypeScript(fieldInfo.fieldType, fieldInfo.isOptional);
       const optional = fieldInfo.alwaysOptional ? "?" : "";
       output += `  readonly ${fieldName}${optional}: ${tsType};\n`;
     }
@@ -252,32 +153,78 @@ function generateInternalTypes(allRecordModels) {
   return output;
 }
 
+/**
+ * Generate per-version internal interfaces (e.g. `ShapeBox__v1`, `ShapeBox__v2`).
+ *
+ * Each interface contains only the fields present in that version, with their
+ * actual optionality taken straight from that version's IR. The downstream
+ * `DocumentUpgradeFns` generator uses these so each upgrade function's
+ * parameter can `Pick` the prior version's fields and its return type can
+ * index the current version's field type precisely.
+ */
+function generatePerVersionInternalTypes(chain) {
+  // Group by model so each model's versions appear together in ascending order.
+  const perModelVersions = new Map();
+  for (const {
+    version,
+    ir
+  } of chain) {
+    for (const [modelKey, modelDef] of Object.entries(ir.models)) {
+      if (modelDef.type !== "record") continue;
+      let list = perModelVersions.get(modelKey);
+      if (list == null) {
+        list = [];
+        perModelVersions.set(modelKey, list);
+      }
+      list.push({
+        version,
+        fields: modelDef.record.fields
+      });
+    }
+  }
+  let output = "";
+  const sortedModels = Array.from(perModelVersions.entries()).sort(([a], [b]) => a.localeCompare(b));
+  for (const [modelKey, versions] of sortedModels) {
+    for (const {
+      version,
+      fields
+    } of versions) {
+      output += `/** Internal representation of ${modelKey} at schema version ${version}. */\n`;
+      output += `export interface ${modelKey}__v${version} {\n`;
+      const sortedFields = [...fields].sort((a, b) => a.key.localeCompare(b.key));
+      for (const field of sortedFields) {
+        const tsType = convertFieldTypeToInternalTypeScript(field.fieldType, field.isOptional === true);
+        const optional = field.isOptional === true ? "?" : "";
+        output += `  readonly ${field.key}${optional}: ${tsType};\n`;
+      }
+      output += "}\n\n";
+    }
+  }
+  return output;
+}
+
 /**
  * Generate _internal/upgrades.ts content.
  */
-function generateUpgrades(allRecordModels, latestSchema) {
+function generateUpgrades(allRecordModels, latestIr) {
   let output = GENERATED_FILE_HEADER;
   const hasRecords = allRecordModels.size > 0;
-  const hasUnions = Object.values(latestSchema).some(isUnionSchema);
+  const hasUnions = Object.values(latestIr.models).some(m => m.type === "union");
   const imports = [];
   if (hasRecords) imports.push("UpgradeRegistry");
   if (hasUnions) imports.push("UnionUpgradeRegistry");
   if (imports.length > 0) {
     output += `import type { ${imports.join(", ")} } from "@palantir/pack.document-schema.model-types";\n\n`;
   }
-  for (const [exportName, model] of sortedEntries(allRecordModels)) {
-    output += `export const ${exportName}Upgrades: UpgradeRegistry<"${exportName}"> = {\n`;
-    output += `  modelName: "${exportName}",\n`;
+  for (const [modelKey, model] of sortedEntries(allRecordModels)) {
+    output += `export const ${modelKey}Upgrades: UpgradeRegistry<"${modelKey}"> = {\n`;
+    output += `  modelName: "${modelKey}",\n`;
 
     // allFields
     output += `  allFields: {\n`;
     for (const [fieldName, fieldInfo] of sortedEntries(model.allFields)) {
-      const typeDescriptor = convertTypeToFieldTypeDescriptor(fieldInfo.fieldType, latestSchema);
-      if (fieldInfo.default !== undefined) {
-        output += `    ${fieldName}: { type: ${typeDescriptor}, default: ${JSON.stringify(fieldInfo.default)} },\n`;
-      } else {
-        output += `    ${fieldName}: { type: ${typeDescriptor} },\n`;
-      }
+      const typeDescriptor = convertFieldTypeToDescriptor(fieldInfo.fieldType, fieldInfo.isOptional);
+      output += `    ${fieldName}: { type: ${typeDescriptor} },\n`;
     }
     output += `  },\n`;
 
@@ -285,13 +232,11 @@ function generateUpgrades(allRecordModels, latestSchema) {
     output += `  steps: [\n`;
     for (const step of model.steps) {
       output += `    {\n`;
-      output += `      name: "${step.name}",\n`;
       output += `      addedInVersion: ${step.addedInVersion},\n`;
       output += `      fields: {\n`;
       for (const [fieldName, fieldMeta] of step.fields) {
         output += `        ${fieldName}: {\n`;
         output += `          derivedFrom: [${fieldMeta.derivedFrom.map(d => `"${d}"`).join(", ")}],\n`;
-        output += `          forward: ${fieldMeta.forwardSource},\n`;
         if (fieldMeta.default !== undefined) {
           output += `          default: ${JSON.stringify(fieldMeta.default)},\n`;
         }
@@ -308,19 +253,15 @@ function generateUpgrades(allRecordModels, latestSchema) {
   }
 
   // Generate UnionUpgradeRegistry entries for union models
-  for (const [exportName, item] of Object.entries(latestSchema)) {
-    if (!isUnionSchema(item)) continue;
-    output += `export const ${exportName}Upgrades: UnionUpgradeRegistry<"${exportName}"> = {\n`;
-    output += `  modelName: "${exportName}",\n`;
-    output += `  discriminant: "${item.discriminant}",\n`;
+  for (const [modelKey, modelDef] of Object.entries(latestIr.models)) {
+    if (modelDef.type !== "union") continue;
+    const union = modelDef.union;
+    output += `export const ${modelKey}Upgrades: UnionUpgradeRegistry<"${modelKey}"> = {\n`;
+    output += `  modelName: "${modelKey}",\n`;
+    output += `  discriminant: "${union.discriminant}",\n`;
     output += `  variants: {\n`;
-    for (const [variantKey, variantField] of Object.entries(item.variants)) {
-      if (variantField.type === "ref" && variantField.refType === "record" && variantField.name != null) {
-        const variantExportName = findRecordExportName(variantField.name, latestSchema);
-        if (variantExportName != null) {
-          output += `    "${variantKey}": "${variantExportName}",\n`;
-        }
-      }
+    for (const [variantKey, variantModelKey] of Object.entries(union.variants)) {
+      output += `    "${variantKey}": "${variantModelKey}",\n`;
     }
     output += `  },\n`;
     output += `};\n\n`;
@@ -334,21 +275,13 @@ function generateUpgrades(allRecordModels, latestSchema) {
 function generateInternalSchemaContent(allRecordModels) {
   let output = GENERATED_FILE_HEADER;
   output += `import { z } from "zod";\n\n`;
-  for (const [exportName, model] of sortedEntries(allRecordModels)) {
-    output += `export const ${exportName}InternalSchema = z.object({\n`;
+  for (const [modelKey, model] of sortedEntries(allRecordModels)) {
+    output += `export const ${modelKey}InternalSchema = z.object({\n`;
     for (const [fieldName, fieldInfo] of sortedEntries(model.allFields)) {
-      const zodType = convertTypeToZodSchema(fieldInfo.fieldType);
-      // In internal schema, all fields that might be absent are optional
-      if (fieldInfo.alwaysOptional) {
-        // If the type already ends with .optional(), don't double-wrap
-        if (zodType.endsWith(".optional()")) {
-          output += `  ${fieldName}: ${zodType},\n`;
-        } else {
-          output += `  ${fieldName}: ${zodType}.optional(),\n`;
-        }
-      } else {
-        output += `  ${fieldName}: ${zodType},\n`;
-      }
+      // Use everOptional: a field optional in ANY version must stay optional
+      // in the internal schema because older documents may legitimately omit it.
+      const zodType = convertFieldTypeToInternalZod(fieldInfo.fieldType, fieldInfo.alwaysOptional || fieldInfo.everOptional);
+      output += `  ${fieldName}: ${zodType},\n`;
     }
     output += `}).passthrough();\n\n`;
   }
@@ -356,25 +289,103 @@ function generateInternalSchemaContent(allRecordModels) {
 }
 
 /**
- * Generate _internal/types.ts, _internal/upgrades.ts, and _internal/schema.ts
- * from a versioned schema chain.
+ * Generate _internal/upgradeFns.ts content: a typed `DocumentUpgradeFns`
+ * interface enumerating every upgrade function the app must supply at boot.
+ *
+ * For every `(modelName, v${version}, fieldName)` triple where `derivedFrom` is
+ * non-empty, emit a property typed
+ * `(oldFields: Pick<<Model>__v<prior>, derivedFrom[number]>) => <Model>__v<current>[fieldName]`.
+ * Additive fields (empty `derivedFrom`) are intentionally absent — they are
+ * handled by the structural `default` in `_internal/upgrades.ts`.
  *
- * @param schema - The schema definition (initial or versioned)
- * @returns Object containing generated code for each internal file
+ * The interface is consumed by the generated `DocumentModel` factory in
+ * `models.ts`, which requires it as a parameter when the schema has any
+ * derived fields. Schemas without derived fields emit an empty interface
+ * (still exported for type-uniformity in downstream code).
+ */
+function generateUpgradeFns(chain, allRecordModels) {
+  const perModel = new Map();
+  const neededTypes = new Set();
+  for (const [modelKey, model] of sortedEntries(allRecordModels)) {
+    for (const step of model.steps) {
+      const currentIdx = chain.findIndex(c => c.version === step.addedInVersion);
+      if (currentIdx <= 0) continue;
+      const priorVersion = chain[currentIdx - 1].version;
+      const fields = [];
+      for (const [fieldName, fieldMeta] of step.fields) {
+        if (fieldMeta.derivedFrom.length === 0) continue;
+        fields.push({
+          fieldName,
+          derivedFrom: [...fieldMeta.derivedFrom]
+        });
+      }
+      if (fields.length === 0) continue;
+      let list = perModel.get(modelKey);
+      if (list == null) {
+        list = [];
+        perModel.set(modelKey, list);
+      }
+      list.push({
+        currentVersion: step.addedInVersion,
+        priorVersion,
+        fields
+      });
+      neededTypes.add(`${modelKey}__v${priorVersion}`);
+      neededTypes.add(`${modelKey}__v${step.addedInVersion}`);
+    }
+  }
+  let output = GENERATED_FILE_HEADER;
+  output += `\n`;
+  if (neededTypes.size > 0) {
+    const sorted = Array.from(neededTypes).sort();
+    output += `import type { ${sorted.join(", ")} } from "./types.js";\n\n`;
+  }
+  output += `/**\n`;
+  output += ` * Typed upgrade functions the application must supply at boot. Coverage is\n`;
+  output += ` * exhaustive: missing or extra entries fail type-check at the \`DocumentModel(...)\`\n`;
+  output += ` * call site. Additive fields (empty \`derivedFrom\`) are intentionally absent.\n`;
+  output += ` */\n`;
+  if (perModel.size === 0) {
+    output += `// eslint-disable-next-line @typescript-eslint/no-empty-object-type\n`;
+    output += `export interface DocumentUpgradeFns {}\n`;
+  } else {
+    output += `export interface DocumentUpgradeFns {\n`;
+    for (const [modelKey, steps] of Array.from(perModel.entries()).sort(([a], [b]) => a.localeCompare(b))) {
+      output += `  ${modelKey}: {\n`;
+      for (const step of steps) {
+        output += `    v${step.currentVersion}: {\n`;
+        for (const f of step.fields) {
+          const pickKeys = f.derivedFrom.map(d => `"${d}"`).join(" | ");
+          output += `      ${f.fieldName}: (oldFields: Pick<${modelKey}__v${step.priorVersion}, ${pickKeys}>) => ${modelKey}__v${step.currentVersion}["${f.fieldName}"];\n`;
+        }
+        output += `    };\n`;
+      }
+      output += `  };\n`;
+    }
+    output += `}\n`;
+  }
+  return output;
+}
+
+/**
+ * Generate _internal/types.ts, _internal/upgrades.ts, _internal/schema.ts and
+ * _internal/upgradeFns.ts from an already-resolved versioned IR chain.
  */
-export function generateInternalFromSchema(schema) {
+export function generateInternalFromChain(resolved) {
   const {
     chain
-  } = resolveSchemaChain(schema);
-  const latestSchema = chain[chain.length - 1].schema;
+  } = resolved;
+  const latestIr = chain[chain.length - 1].ir;
   const allRecordModels = collectRecordModels(chain);
-  const internalTypes = generateInternalTypes(allRecordModels);
-  const upgrades = generateUpgrades(allRecordModels, latestSchema);
+  const internalTypes = generateInternalTypes(allRecordModels) + (chainHasDerivedFields(chain) ? generatePerVersionInternalTypes(chain) : "");
+  const upgrades = generateUpgrades(allRecordModels, latestIr);
   const internalSchema = generateInternalSchemaContent(allRecordModels);
+  const internalUpgradeFns = generateUpgradeFns(chain, allRecordModels);
   return {
     internalTypes,
     upgrades,
-    internalSchema
+    internalSchema,
+    internalUpgradeFns
   };
 }
 //# sourceMappingURL=generateInternalFromSchema.js.map