@pdtf/schemas 3.6.0-4 → 3.6.0-41

package/src/utils/extractExtensionOverlays.js

@@ -5,6 +5,41 @@ const path = require("path");
 
 const combinedSchema = require("../schemas/v3/combined.json");
 
+// Derive ref-related field names from a refType
+function getRefConfig(refType) {
+  if (refType === "nts2Ref") {
+    // Legacy mapping: nts2Ref -> ntsRef in output
+    return {
+      ref: "nts2Ref",
+      outputRef: "ntsRef",
+      required: "nts2Required",
+      title: "nts2Title",
+      description: "nts2Description",
+      enumKey: "nts2Enum",
+    };
+  }
+  if (refType === "sef25Ref") {
+    // SEF25 extensions also appear as ntsRef in output
+    return {
+      ref: "sef25Ref",
+      outputRef: "ntsRef",
+      required: "sef25Required",
+      title: "sef25Title",
+      description: "sef25Description",
+      enumKey: "sef25Enum",
+    };
+  }
+  // For other ref types, derive keys by replacing "Ref" suffix
+  return {
+    ref: refType,
+    outputRef: refType,
+    required: refType.replace("Ref", "Required"),
+    title: refType.replace("Ref", "Title"),
+    description: refType.replace("Ref", "Description"),
+    enumKey: refType.replace("Ref", "Enum"),
+  };
+}
+
 // Define the extension overlay mappings
 const extensionMappings = {
   // Outside areas
@@ -150,18 +185,140 @@ const extensionMappings = {
       "/properties/propertyPack/properties/completionAndMoving/properties/otherPropertyInChain",
     ],
   },
+
+  // Private supply costs (water & sewerage)
+  sc: {
+    name: "supplyCosts",
+    description: "Associated costs for private water and sewerage supply",
+    refType: "sef25Ref",
+    paths: [
+      "/properties/propertyPack/properties/waterAndDrainage/properties/water/properties/mainsWater/oneOf/0/properties/associatedCost",
+      "/properties/propertyPack/properties/waterAndDrainage/properties/drainage/properties/mainsFoulDrainage/oneOf/2/properties/associatedCost",
+    ],
+  },
+
+  // Parking permit costs
+  pc: {
+    name: "parkingPermitCost",
+    description: "Parking permit cost and frequency",
+    refType: "sef25Ref",
+    paths: [
+      "/properties/propertyPack/properties/parking/properties/controlledParking/oneOf/1/properties/costOfPermit",
+      "/properties/propertyPack/properties/parking/properties/controlledParking/oneOf/1/properties/costOfPermitFrequency",
+    ],
+  },
+
+  // Property hazards
+  ph: {
+    name: "propertyHazards",
+    description: "Property hazards and known issues",
+    refType: "sef25Ref",
+    paths: [
+      "/properties/propertyPack/properties/specialistIssues",
+    ],
+  },
+
+  // Dropped kerb access to parking
+  dk: {
+    name: "droppedKerbAccess",
+    description: "Dropped kerb access to parking",
+    refType: "sef25Ref",
+    paths: [
+      "/properties/propertyPack/properties/parking/properties/droppedKerbAccess",
+    ],
+  },
+
+  // Private right of way
+  rw: {
+    name: "privateRightOfWay",
+    description: "Private right of way",
+    refType: "sef25Ref",
+    paths: [
+      "/properties/propertyPack/properties/rightsAndInformalArrangements/properties/rightsOrArrangements/properties/privateRightOfWay",
+    ],
+  },
+
+  // Storm, fire or flood damage
+  sd: {
+    name: "stormFireFloodDamage",
+    description: "Storm, fire or flood damage",
+    refType: "sef25Ref",
+    paths: [
+      "/properties/propertyPack/properties/environmentalIssues/properties/stormFireFloodDamage",
+    ],
+  },
+
+  // Solar panel lease costs
+  lc: {
+    name: "solarLeaseCosts",
+    description: "Solar panel lease costs",
+    refType: "sef25Ref",
+    paths: [
+      "/properties/propertyPack/properties/electricity/properties/solarPanels/oneOf/1/properties/panelsOwnedOutright/oneOf/1/properties/associatedCost",
+    ],
+  },
+
+  // Warranties and guarantees upfront
+  wg: {
+    name: "warrantiesGuarantees",
+    description: "Warranties and guarantees upfront Y/N for SEF25",
+    refType: "sef25Ref",
+    paths: [
+      "/properties/propertyPack/properties/guaranteesWarrantiesAndIndemnityInsurances",
+    ],
+  },
+
+  // Insurance claims upfront
+  ic: {
+    name: "insuranceClaims",
+    description: "Insurance claims upfront for SEF25",
+    refType: "sef25Ref",
+    paths: [
+      "/properties/propertyPack/properties/insurance/oneOf/1/properties/insuranceClaims",
+    ],
+  },
+
+  // Neighbour development upfront
+  nd: {
+    name: "neighbourDevelopment",
+    description: "Neighbour development upfront for SEF25",
+    refType: "sef25Ref",
+    paths: [
+      "/properties/propertyPack/properties/notices/properties/neighbourDevelopment",
+    ],
+  },
+
+  // Other material issue upfront
+  mi: {
+    name: "otherMaterialIssue",
+    description: "Other material issue upfront for SEF25",
+    refType: "sef25Ref",
+    paths: [
+      "/properties/propertyPack/properties/additionalInformation/properties/otherMaterialIssue",
+    ],
+  },
+
+  // Title restrictions for freehold
+  tr: {
+    name: "titleRestrictions",
+    description: "Title restrictions for freehold properties",
+    refType: "sef25Ref",
+    paths: [
+      "/properties/propertyPack/properties/ownership/properties/ownershipsToBeTransferred/items/oneOf/0/properties/titleRestrictions",
+    ],
+  },
 };
 
-// Function to extract properties at specific paths with nts2Ref
-function extractPathsWithNts2Ref(schema, paths) {
+// Function to extract properties at specific paths with given ref type
+function extractPathsWithRef(schema, paths, refConfig) {
   const result = {};
 
   paths.forEach((path) => {
     try {
       const value = jp.get(schema, path);
-      if (value && hasNts2Ref(value)) {
+      if (value && hasRef(value, refConfig)) {
         // Set the value at the same path in result
-        jp.set(result, path, extractNts2Properties(value));
+        jp.set(result, path, extractRefProperties(value, refConfig));
       }
     } catch (err) {
       console.warn(`Path not found: ${path}`);
@@ -171,11 +328,11 @@ function extractPathsWithNts2Ref(schema, paths) {
   return result;
 }
 
-// Check if an object or its descendants have nts2Ref
-function hasNts2Ref(obj) {
+// Check if an object or its descendants have the specified ref
+function hasRef(obj, refConfig) {
   let found = false;
   traverse(obj).forEach(function (element) {
-    if (element && element.nts2Ref) {
+    if (element && element[refConfig.ref]) {
       found = true;
       this.stop();
     }
@@ -183,16 +340,16 @@ function hasNts2Ref(obj) {
   return found;
 }
 
-// Extract only NTS2-specific properties (with nts2Ref)
-function extractNts2Properties(obj, parentKey) {
+// Extract only ref-specific properties
+function extractRefProperties(obj, refConfig, parentKey) {
   const result = {};
 
-  // Copy nts2-specific metadata at current level
-  if (obj.nts2Ref) result.ntsRef = obj.nts2Ref;
-  if (obj.nts2Required) result.required = obj.nts2Required;
-  if (obj.nts2Title) result.title = obj.nts2Title;
-  if (obj.nts2Description) result.description = obj.nts2Description;
-  if (obj.nts2Enum) result.enum = obj.nts2Enum;
+  // Copy ref-specific metadata at current level
+  if (obj[refConfig.ref]) result[refConfig.outputRef] = obj[refConfig.ref];
+  if (obj[refConfig.required]) result.required = obj[refConfig.required];
+  if (obj[refConfig.title]) result.title = obj[refConfig.title];
+  if (obj[refConfig.description]) result.description = obj[refConfig.description];
+  if (obj[refConfig.enumKey]) result.enum = obj[refConfig.enumKey];
 
   // Handle discriminator
   if (obj.discriminator) {
@@ -204,11 +361,11 @@ function extractNts2Properties(obj, parentKey) {
     result.properties = {};
     Object.keys(obj.properties).forEach((key) => {
       const prop = obj.properties[key];
-      if (hasNts2Ref(prop)) {
-        result.properties[key] = extractNts2Properties(prop, key);
+      if (hasRef(prop, refConfig)) {
+        result.properties[key] = extractRefProperties(prop, refConfig, key);
       }
     });
-    // Only keep properties if we found some with nts2Ref
+    // Only keep properties if we found some with ref
     if (Object.keys(result.properties).length === 0) {
       delete result.properties;
     }
@@ -216,8 +373,8 @@ function extractNts2Properties(obj, parentKey) {
 
   // Handle arrays
   if (obj.items) {
-    if (hasNts2Ref(obj.items)) {
-      result.items = extractNts2Properties(obj.items);
+    if (hasRef(obj.items, refConfig)) {
+      result.items = extractRefProperties(obj.items, refConfig);
     }
   }
 
@@ -225,8 +382,8 @@ function extractNts2Properties(obj, parentKey) {
   if (obj.oneOf) {
     // Always preserve the full array structure and order
     const processedOneOf = obj.oneOf.map((schema, index) => {
-      if (hasNts2Ref(schema)) {
-        const extracted = extractNts2Properties(schema);
+      if (hasRef(schema, refConfig)) {
+        const extracted = extractRefProperties(schema, refConfig);
         // For oneOf schemas, we need to preserve discriminator enum values
         if (obj.discriminator && schema.properties) {
           const propName = obj.discriminator.propertyName;
@@ -234,7 +391,7 @@ function extractNts2Properties(obj, parentKey) {
             if (!extracted.properties) extracted.properties = {};
             extracted.properties[propName] = {
               enum:
-                schema.properties[propName].nts2Enum ||
+                schema.properties[propName][refConfig.enumKey] ||
                 schema.properties[propName].enum,
             };
           }
@@ -258,7 +415,7 @@ function extractNts2Properties(obj, parentKey) {
         }
         return extracted;
       } else {
-        // If this branch doesn't have nts2Ref, preserve the base discriminator enum
+        // If this branch doesn't have ref, preserve the base discriminator enum
         if (obj.discriminator && schema.properties) {
           const propName = obj.discriminator.propertyName;
           if (schema.properties[propName]) {
@@ -281,7 +438,7 @@ function extractNts2Properties(obj, parentKey) {
     // If parentKey is set, and this object has extension metadata, return a stub for the parent
     if (
       parentKey &&
-      (result.ntsRef || result.required || result.discriminator)
+      (result[refConfig.outputRef] || result.required || result.discriminator)
     ) {
       // This is an extension property inside a oneOf branch
       // Return a stub for the parent property with metadata and oneOf structure
@@ -311,16 +468,16 @@ function extractNts2Properties(obj, parentKey) {
 }
 
 // Add required array at parent levels if needed
-function addRequiredArrays(overlay, schema) {
+function addRequiredArrays(overlay, schema, refConfig) {
   traverse(overlay).forEach(function (node) {
     if (node && node.properties) {
       const schemaPath = "/" + this.path.join("/");
       try {
         const schemaNode = jp.get(schema, schemaPath);
-        if (schemaNode && schemaNode.nts2Required) {
+        if (schemaNode && schemaNode[refConfig.required]) {
           // Filter to only include properties that exist in this overlay
           const overlayProps = Object.keys(node.properties);
-          const requiredProps = schemaNode.nts2Required.filter((prop) =>
+          const requiredProps = schemaNode[refConfig.required].filter((prop) =>
             overlayProps.includes(prop)
           );
           if (requiredProps.length > 0) {
@@ -335,15 +492,43 @@ function addRequiredArrays(overlay, schema) {
   return overlay;
 }
 
+// Annotate intermediate nodes with refs from the source schema
+// This ensures every level of the overlay tree carries an ntsRef
+function addIntermediateRefs(overlay, schema, refConfig) {
+  traverse(overlay).forEach(function (node) {
+    if (node && typeof node === "object" && !Array.isArray(node) && node.properties) {
+      // Skip if this node already has a ref
+      if (node[refConfig.outputRef]) return;
+
+      const schemaPath = "/" + this.path.join("/");
+      try {
+        const schemaNode = jp.get(schema, schemaPath);
+        if (schemaNode && schemaNode[refConfig.ref]) {
+          node[refConfig.outputRef] = schemaNode[refConfig.ref];
+        }
+      } catch (err) {
+        // Path doesn't exist in schema, skip
+      }
+    }
+  });
+  return overlay;
+}
+
 // Generate extension overlays
 Object.entries(extensionMappings).forEach(([code, config]) => {
-  console.log(`\nGenerating extension overlay: ${code} (${config.name})`);
+  const refType = config.refType || "nts2Ref";
+  const refConfig = getRefConfig(refType);
+
+  console.log(`\nGenerating extension overlay: ${code} (${config.name}) [${refType}]`);
 
   // Extract the specific paths
-  let overlay = extractPathsWithNts2Ref(combinedSchema, config.paths);
+  let overlay = extractPathsWithRef(combinedSchema, config.paths, refConfig);
 
   // Add required arrays where needed
-  overlay = addRequiredArrays(overlay, combinedSchema);
+  overlay = addRequiredArrays(overlay, combinedSchema, refConfig);
+
+  // Add refs to intermediate nodes in the path hierarchy
+  overlay = addIntermediateRefs(overlay, combinedSchema, refConfig);
 
   // Add schema metadata
   overlay.$schema = "http://json-schema.org/draft-07/schema#";

package/src/utils/extractOverlay.js

@@ -2,9 +2,24 @@ const { dereference } = require("@jdw/jst");
 const traverse = require("traverse");
 const jp = require("jsonpointer");
 const fs = require("fs");
+const path = require("path");
 const merge = require("deepmerge");
 
-const combinedSchema = require("../schemas/v3/combined.json");
+const combinedPath =
+  process.env.COMBINED_PATH ||
+  path.resolve(__dirname, "../schemas/v3/combined.json");
+const outputDir =
+  process.env.OUTPUT_DIR ||
+  path.resolve(__dirname, "../schemas/v3");
+
+if (!fs.existsSync(combinedPath)) {
+  throw new Error(`Combined schema not found: ${combinedPath}`);
+}
+if (!fs.existsSync(outputDir)) {
+  fs.mkdirSync(outputDir, { recursive: true });
+}
+
+const combinedSchema = JSON.parse(fs.readFileSync(combinedPath, "utf8"));
 
 const extractFields = [
   "baspi4",
@@ -14,7 +29,9 @@ const extractFields = [
   "ntsl",
   "ntsl2",
   "ta6",
+  "ta6ed6",
   "ta7",
+  "ta7ed5",
   "ta10",
   "lpe1",
   "fme1",
@@ -25,6 +42,7 @@ const extractFields = [
   "rds",
   "oc1",
   "sr24",
+  "sef25",
 ];
 
 const flattenSkeleton = (schema) => {
@@ -68,23 +86,21 @@ const flattenSkeleton = (schema) => {
     });
   }
 
-  // Handle oneOf - merge all possible properties
+  // Handle oneOf - merge all possible properties (recursively handles nested oneOf)
   if (schema.oneOf) {
     schema.oneOf.forEach((aOneOf) => {
-      if (aOneOf.properties) {
-        Object.entries(aOneOf.properties).forEach(([key, value]) => {
-          // If property already exists and differs, mark as variant
-          const newValue = flattenSkeleton(value);
-          if (returnStructure[key] && JSON.stringify(returnStructure[key]) !== JSON.stringify(newValue)) {
-            // For primitive types, just mark as variant type
-            if (typeof returnStructure[key] === "string" || typeof newValue === "string") {
+      const variantResult = flattenSkeleton(aOneOf);
+      if (variantResult && typeof variantResult === "object" && !Array.isArray(variantResult)) {
+        Object.entries(variantResult).forEach(([key, value]) => {
+          if (key === "_variants") return;
+          if (returnStructure[key] && JSON.stringify(returnStructure[key]) !== JSON.stringify(value)) {
+            if (typeof returnStructure[key] === "string" || typeof value === "string") {
               returnStructure[key] = "variant";
             } else {
-              // For complex types, merge properties
-              returnStructure[key] = { ...returnStructure[key], ...newValue, _variants: true };
+              returnStructure[key] = { ...returnStructure[key], ...value, _variants: true };
             }
           } else {
-            returnStructure[key] = newValue;
+            returnStructure[key] = value;
           }
         });
       }
@@ -97,6 +113,23 @@ const flattenSkeleton = (schema) => {
 
 const extractOverlay = (sourceSchema, ref) => {
   const refName = `${ref}Ref`;
+  const extendedTagMap = {
+    Type: "type",
+    MinLength: "minLength",
+    MaxLength: "maxLength",
+    Pattern: "pattern",
+    Minimum: "minimum",
+    Maximum: "maximum",
+    MinItems: "minItems",
+    MaxItems: "maxItems",
+    UniqueItems: "uniqueItems",
+    AdditionalProperties: "additionalProperties",
+    Format: "format",
+    MinProperties: "minProperties",
+    MaxProperties: "maxProperties",
+    MultipleOf: "multipleOf",
+    Const: "const",
+  };
   const returnSchema = {
     $schema: "http://json-schema.org/draft-07/schema#",
     $id: `https://trust.propdata.org.uk/schemas/v3/overlays/${ref}.json`,
@@ -107,7 +140,7 @@ const extractOverlay = (sourceSchema, ref) => {
     if (element[refName]) {
       jp.set(returnSchema, `${path}/${refName}`, element[refName]);
 
-      if (element.discriminator) {
+      if (element.discriminator && Array.isArray(element.oneOf)) {
         jp.set(returnSchema, `${path}/discriminator`, element.discriminator);
         const { propertyName } = element.discriminator;
         element.oneOf.forEach((oneOf, index) => {
@@ -134,7 +167,7 @@ const extractOverlay = (sourceSchema, ref) => {
       }
 
       // also handle discriminator properties nested in items
-      if (element.items?.discriminator) {
+      if (element.items?.discriminator && Array.isArray(element.items.oneOf)) {
         jp.set(
           returnSchema,
           `${path}/items/discriminator`,
@@ -184,6 +217,18 @@ const extractOverlay = (sourceSchema, ref) => {
     if (element[refEnum]) {
       jp.set(returnSchema, `${path}/enum`, element[refEnum]);
     }
+
+    const refConst = `${ref}Const`;
+    if (Object.prototype.hasOwnProperty.call(element, refConst)) {
+      jp.set(returnSchema, `${path}/const`, element[refConst]);
+    }
+
+    Object.entries(extendedTagMap).forEach(([suffix, schemaKey]) => {
+      const tagKey = `${ref}${suffix}`;
+      if (Object.prototype.hasOwnProperty.call(element, tagKey)) {
+        jp.set(returnSchema, `${path}/${schemaKey}`, element[tagKey]);
+      }
+    });
   });
   return returnSchema;
 };
@@ -198,11 +243,15 @@ const deleteProperties = (sourceSchema, propertyNames) => {
 };
 
 const overlays = {};
+const overlaysDir = path.join(outputDir, "overlays");
+if (!fs.existsSync(overlaysDir)) {
+  fs.mkdirSync(overlaysDir, { recursive: true });
+}
 
 extractFields.forEach((key) => {
   let overlay = extractOverlay(combinedSchema, key);
   overlays[key] = overlay;
-  const fileName = `../schemas/v3/overlays/${key}.json`;
+  const fileName = path.join(overlaysDir, `${key}.json`);
   fs.writeFileSync(fileName, JSON.stringify(overlay, null, 2));
   console.log(`Overlay ${key} written to ${fileName}`);
 });
@@ -212,14 +261,30 @@ const coreSchema = deleteProperties(combinedSchema, [
   ...extractFields.map((item) => `${item}Ref`),
   ...extractFields.map((item) => `${item}Required`),
   ...extractFields.map((item) => `${item}Title`),
+  ...extractFields.map((item) => `${item}Description`),
   ...extractFields.map((item) => `${item}Enum`),
+  ...extractFields.map((item) => `${item}Type`),
+  ...extractFields.map((item) => `${item}MinLength`),
+  ...extractFields.map((item) => `${item}MaxLength`),
+  ...extractFields.map((item) => `${item}Pattern`),
+  ...extractFields.map((item) => `${item}Minimum`),
+  ...extractFields.map((item) => `${item}Maximum`),
+  ...extractFields.map((item) => `${item}MinItems`),
+  ...extractFields.map((item) => `${item}MaxItems`),
+  ...extractFields.map((item) => `${item}UniqueItems`),
+  ...extractFields.map((item) => `${item}AdditionalProperties`),
+  ...extractFields.map((item) => `${item}Format`),
+  ...extractFields.map((item) => `${item}MinProperties`),
+  ...extractFields.map((item) => `${item}MaxProperties`),
+  ...extractFields.map((item) => `${item}MultipleOf`),
+  ...extractFields.map((item) => `${item}Const`),
 ]);
 
 fs.writeFileSync(
-  "../schemas/v3/pdtf-transaction.json",
+  path.join(outputDir, "pdtf-transaction.json"),
   JSON.stringify(coreSchema, null, 2)
 );
-console.log("Core schema written to ../schemas/v3/pdtf-transaction.json");
+console.log(`Core schema written to ${path.join(outputDir, "pdtf-transaction.json")}`);
 
 const skeletonSchema = deleteProperties(coreSchema, [
   "$schema",
@@ -238,10 +303,10 @@ const skeletonSchema = deleteProperties(coreSchema, [
 const skeletonSchemaFlattened = flattenSkeleton(skeletonSchema);
 
 fs.writeFileSync(
-  "../schemas/v3/skeleton.json",
+  path.join(outputDir, "skeleton.json"),
   JSON.stringify(skeletonSchemaFlattened, null, 2)
 );
-console.log("Flat Skeleton schema written to ../schemas/v3/skeleton.json");
+console.log(`Flat Skeleton schema written to ${path.join(outputDir, "skeleton.json")}`);
 
 // Generate compact skeleton format for better token efficiency
 const toCompact = (obj, indent = "") => {
@@ -267,8 +332,8 @@ const toCompact = (obj, indent = "") => {
     // Check if this is a keyed object (has "*" key)
     if (keys.length === 1 && keys[0] === "*") {
       const inner = toCompact(obj["*"], indent);
-      if (inner === "") {
-        return "{*}"; // Keyed object of primitives
+      if (inner === "" || inner === "{*}") {
+        return "{*}"; // Keyed object of primitives (or nested keyed objects with no structure)
       }
       return `{*}\n${inner}`;
     }
@@ -308,7 +373,7 @@ const innerContent = toCompact(skeletonSchemaFlattened, "");
 const compactSkeleton = innerContent ? innerContent : "";
 
 fs.writeFileSync(
-  "../schemas/v3/compactSkeleton.txt",
+  path.join(outputDir, "compactSkeleton.txt"),
   compactSkeleton
 );
-console.log("Compact Skeleton written to ../schemas/v3/compactSkeleton.txt");
+console.log(`Compact Skeleton written to ${path.join(outputDir, "compactSkeleton.txt")}`);

package/src/utils/pathSkeleton.js

@@ -6,7 +6,7 @@ const combinedSchema = require("../schemas/v3/combined.json");
 // Extract fields that have overlay-specific properties
 const extractFields = [
   "baspi4", "baspi5", "nts", "nts2", "ntsl", "ntsl2",
-  "ta6", "ta7", "ta10", "lpe1", "fme1", "piq",
+  "ta6", "ta7", "ta7ed5", "ta10", "lpe1", "fme1", "piq",
   "con29R", "con29DW", "llc1", "rds", "oc1", "sr24",
 ];