@pdtf/schemas 3.6.0-dev.3 → 3.6.0-dev.4

package/keen-dewdney-6ae15b/src/utils/compactSkeleton.js

@@ -0,0 +1,84 @@
+const fs = require("fs");
+
+// Read the current skeleton
+const skeleton = require("../schemas/v3/skeleton.json");
+
+// Convert to ultra-compact format
+const toCompact = (obj, indent = "") => {
+  if (typeof obj === "string") {
+    // Type indicators
+    if (["string", "number", "integer", "boolean", "variant"].includes(obj)) {
+      return obj[0]; // Just first letter: s, n, i, b, v
+    }
+    return obj;
+  }
+  
+  if (Array.isArray(obj)) {
+    if (obj.length === 0) return "[]";
+    // Array with content
+    const inner = toCompact(obj[0], indent + "  ");
+    if (typeof inner === "string" && inner.length === 1) {
+      return `[${inner}]`; // Simple array like [s]
+    }
+    return `[\n${indent}  ${inner}\n${indent}]`;
+  }
+  
+  if (obj && typeof obj === "object") {
+    const keys = Object.keys(obj);
+    if (keys.length === 0) return "{}";
+    
+    // Check if it's a simple object with only type indicators
+    const allSimple = keys.every(k => 
+      typeof obj[k] === "string" && 
+      ["s", "n", "i", "b", "v"].includes(toCompact(obj[k]))
+    );
+    
+    if (allSimple && keys.length < 5) {
+      // Inline format: {a:s b:n c:s}
+      return `{${keys.map(k => `${k}:${toCompact(obj[k])}`).join(" ")}}`;
+    }
+    
+    // Multi-line format
+    const lines = keys.map(k => {
+      const val = toCompact(obj[k], indent + "  ");
+      if (typeof val === "string" && val.length === 1) {
+        return `${indent}  ${k}:${val}`;
+      } else if (typeof val === "string" && (val === "{}" || val === "[]")) {
+        return `${indent}  ${k}${val}`;
+      } else if (typeof val === "string" && val.startsWith("{") && val.endsWith("}")) {
+        return `${indent}  ${k}${val}`;
+      } else if (typeof val === "string" && val.startsWith("[") && !val.includes("\n")) {
+        return `${indent}  ${k}${val}`;
+      } else {
+        return `${indent}  ${k}:\n${indent}  ${val}`;
+      }
+    });
+    
+    return `{\n${lines.join("\n")}\n${indent}}`;
+  }
+  
+  return String(obj);
+};
+
+// Generate ultra-compact version
+const compact = toCompact(skeleton);
+
+// Write compact version
+fs.writeFileSync("../schemas/v3/compactSkeleton.txt", compact);
+
+console.log("Compact skeleton written to ../schemas/v3/compactSkeleton.txt");
+console.log(`Size: ${compact.length} bytes`);
+
+// Count tokens
+try {
+  const GPT3Encoder = require('gpt-3-encoder');
+  const encoded = GPT3Encoder.encode(compact);
+  console.log(`Tokens (GPT-3): ${encoded.length}`);
+  console.log(`Reduction: ${((1 - encoded.length / 78734) * 100).toFixed(1)}% fewer tokens than JSON skeleton`);
+  
+  // Show sample
+  console.log("\nFirst 500 chars:");
+  console.log(compact.substring(0, 500));
+} catch (e) {
+  console.log(`Estimated tokens: ~${Math.ceil(compact.length / 4)}`);
+}

package/keen-dewdney-6ae15b/src/utils/countTokens.js

@@ -0,0 +1,67 @@
+const fs = require('fs');
+
+// Try to use tiktoken or a similar library
+// For Claude/Anthropic models, we can estimate using the cl100k_base encoding
+// which is similar to what Claude uses
+
+// First, let's check what's available
+try {
+  // Option 1: Use gpt-tokenizer (common npm package)
+  const { encode } = require('gpt-tokenizer');
+  
+  const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+  const tokens = encode(content);
+  
+  console.log(`Using gpt-tokenizer:`);
+  console.log(`Total tokens: ${tokens.length}`);
+  console.log(`File size: ${content.length} characters`);
+  console.log(`Ratio: ${(content.length / tokens.length).toFixed(2)} characters per token`);
+  
+} catch (e1) {
+  console.log("gpt-tokenizer not found, trying gpt-3-encoder...");
+  
+  try {
+    // Option 2: Use gpt-3-encoder
+    const GPT3Encoder = require('gpt-3-encoder');
+    
+    const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+    const encoded = GPT3Encoder.encode(content);
+    
+    console.log(`Using gpt-3-encoder:`);
+    console.log(`Total tokens: ${encoded.length}`);
+    console.log(`File size: ${content.length} characters`);
+    console.log(`Ratio: ${(content.length / encoded.length).toFixed(2)} characters per token`);
+    
+  } catch (e2) {
+    console.log("No tokenizer library found. Falling back to estimation...");
+    
+    // Option 3: Manual estimation
+    const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+    
+    // Count different types of content for better estimation
+    const lines = content.split('\n');
+    const words = content.split(/\s+/);
+    const punctuation = content.match(/[{}:,\[\]"]/g) || [];
+    
+    // Rough estimation:
+    // - JSON structure tokens (brackets, colons, quotes): ~1 token each
+    // - Words: ~1 token each
+    // - Whitespace: usually absorbed into adjacent tokens
+    
+    const structureTokens = punctuation.length;
+    const wordTokens = words.length;
+    const estimatedTokens = Math.ceil((structureTokens + wordTokens) * 0.75); // 0.75 factor for overlap
+    
+    console.log("Manual estimation:");
+    console.log(`File size: ${content.length} characters`);
+    console.log(`Lines: ${lines.length}`);
+    console.log(`Words: ${words.length}`);
+    console.log(`JSON punctuation: ${punctuation.length}`);
+    console.log(`Estimated tokens: ~${estimatedTokens}`);
+    console.log(`Ratio: ${(content.length / estimatedTokens).toFixed(2)} characters per token`);
+    
+    // Also do a simple character-based estimation
+    const simpleEstimate = Math.ceil(content.length / 4);
+    console.log(`\nSimple estimation (chars/4): ~${simpleEstimate} tokens`);
+  }
+}

package/keen-dewdney-6ae15b/src/utils/extractExtensionOverlays.js

@@ -0,0 +1,362 @@
+const traverse = require("traverse");
+const jp = require("jsonpointer");
+const fs = require("fs");
+const path = require("path");
+
+const combinedSchema = require("../schemas/v3/combined.json");
+
+// Define the extension overlay mappings
+const extensionMappings = {
+  // Outside areas
+  oa: {
+    name: "outsideAreas",
+    description: "Outside areas extension for NTS",
+    paths: [
+      "/properties/propertyPack/properties/residentialPropertyFeatures/properties/outsideAreas",
+    ],
+  },
+
+  // Estate rentcharges for freehold
+  er: {
+    name: "estateRentcharges",
+    description: "Estate rentcharges for freehold properties",
+    paths: [
+      "/properties/propertyPack/properties/ownership/properties/ownershipsToBeTransferred/items/oneOf/0/properties/estateRentcharges",
+    ],
+  },
+
+  // Managing agent contact details for leasehold
+  ma: {
+    name: "managingAgent",
+    description: "Managing agent contact details for leasehold",
+    paths: [
+      "/properties/propertyPack/properties/ownership/properties/ownershipsToBeTransferred/items/oneOf/2/properties/leaseholdInformation/properties/contactDetails",
+    ],
+  },
+
+  // Transfer fees for leasehold
+  tf: {
+    name: "transferFees",
+    description: "Transfer fees for leasehold",
+    paths: [
+      "/properties/propertyPack/properties/ownership/properties/ownershipsToBeTransferred/items/oneOf/2/properties/leaseholdInformation/properties/serviceCharge/oneOf/1/properties/transferFees",
+    ],
+  },
+
+  // Main construction type if standard
+  mc: {
+    name: "mainConstruction",
+    description: "Main construction type if standard form",
+    paths: [
+      "/properties/propertyPack/properties/typeOfConstruction/properties/isStandardForm/oneOf/0/properties/constructionType",
+    ],
+  },
+
+  // Loft access and details
+  la: {
+    name: "loftAccess",
+    description: "Loft access and details",
+    paths: [
+      "/properties/propertyPack/properties/typeOfConstruction/properties/loft",
+    ],
+  },
+
+  // Spray foam insulation
+  sf: {
+    name: "sprayFoam",
+    description: "Spray foam insulation",
+    paths: [
+      "/properties/propertyPack/properties/typeOfConstruction/properties/sprayFoamInsulation",
+    ],
+  },
+
+  // Specialist issues - dry rot
+  dr: {
+    name: "dryRot",
+    description: "Dry rot treatment specialist issue",
+    paths: [
+      "/properties/propertyPack/properties/specialistIssues/properties/dryRotEtcTreatment",
+    ],
+  },
+
+  // Specialist issues - asbestos
+  as: {
+    name: "asbestos",
+    description: "Asbestos specialist issue",
+    paths: [
+      "/properties/propertyPack/properties/specialistIssues/properties/containsAsbestos",
+    ],
+  },
+
+  // Specialist issues - Japanese knotweed
+  jk: {
+    name: "japaneseKnotweed",
+    description: "Japanese knotweed specialist issue",
+    paths: [
+      "/properties/propertyPack/properties/specialistIssues/properties/japaneseKnotweed",
+    ],
+  },
+
+  // Specialist issues - subsidence
+  sb: {
+    name: "subsidence",
+    description: "Subsidence or structural fault specialist issue",
+    paths: [
+      "/properties/propertyPack/properties/specialistIssues/properties/subsidenceOrStructuralFault",
+    ],
+  },
+
+  // Specialist issues - health and safety
+  hs: {
+    name: "healthSafety",
+    description: "Health and safety specialist issue",
+    paths: [
+      "/properties/propertyPack/properties/specialistIssues/properties/ongoingHealthOrSafetyIssue",
+    ],
+  },
+
+  // Solar panels leased
+  sl: {
+    name: "solarPanelsLeased",
+    description: "Solar panels ownership details",
+    paths: [
+      "/properties/propertyPack/properties/electricity/properties/solarPanels/oneOf/1/properties/panelsOwnedOutright",
+    ],
+  },
+
+  // Heating installation date
+  hi: {
+    name: "heatingInstalled",
+    description: "Central heating installation date",
+    paths: [
+      "/properties/propertyPack/properties/heating/properties/heatingSystem/oneOf/1/properties/centralHeatingDetails/properties/centralHeatingInstalled",
+    ],
+  },
+
+  // Flood defences
+  fd: {
+    name: "floodDefences",
+    description: "Flood defence information",
+    paths: [
+      "/properties/propertyPack/properties/environmentalIssues/properties/flooding/properties/floodDefences",
+    ],
+  },
+
+  // Other property in chain
+  oc: {
+    name: "otherPropertyChain",
+    description: "Other property in chain dependency",
+    paths: [
+      "/properties/propertyPack/properties/completionAndMoving/properties/otherPropertyInChain",
+    ],
+  },
+};
+
+// Function to extract properties at specific paths with nts2Ref
+function extractPathsWithNts2Ref(schema, paths) {
+  const result = {};
+
+  paths.forEach((path) => {
+    try {
+      const value = jp.get(schema, path);
+      if (value && hasNts2Ref(value)) {
+        // Set the value at the same path in result
+        jp.set(result, path, extractNts2Properties(value));
+      }
+    } catch (err) {
+      console.warn(`Path not found: ${path}`);
+    }
+  });
+
+  return result;
+}
+
+// Check if an object or its descendants have nts2Ref
+function hasNts2Ref(obj) {
+  let found = false;
+  traverse(obj).forEach(function (element) {
+    if (element && element.nts2Ref) {
+      found = true;
+      this.stop();
+    }
+  });
+  return found;
+}
+
+// Extract only NTS2-specific properties (with nts2Ref)
+function extractNts2Properties(obj, parentKey) {
+  const result = {};
+
+  // Copy nts2-specific metadata at current level
+  if (obj.nts2Ref) result.ntsRef = obj.nts2Ref;
+  if (obj.nts2Required) result.required = obj.nts2Required;
+  if (obj.nts2Title) result.title = obj.nts2Title;
+  if (obj.nts2Description) result.description = obj.nts2Description;
+  if (obj.nts2Enum) result.enum = obj.nts2Enum;
+
+  // Handle discriminator
+  if (obj.discriminator) {
+    result.discriminator = obj.discriminator;
+  }
+
+  // Recursively process properties
+  if (obj.properties) {
+    result.properties = {};
+    Object.keys(obj.properties).forEach((key) => {
+      const prop = obj.properties[key];
+      if (hasNts2Ref(prop)) {
+        result.properties[key] = extractNts2Properties(prop, key);
+      }
+    });
+    // Only keep properties if we found some with nts2Ref
+    if (Object.keys(result.properties).length === 0) {
+      delete result.properties;
+    }
+  }
+
+  // Handle arrays
+  if (obj.items) {
+    if (hasNts2Ref(obj.items)) {
+      result.items = extractNts2Properties(obj.items);
+    }
+  }
+
+  // Handle oneOf
+  if (obj.oneOf) {
+    // Always preserve the full array structure and order
+    const processedOneOf = obj.oneOf.map((schema, index) => {
+      if (hasNts2Ref(schema)) {
+        const extracted = extractNts2Properties(schema);
+        // For oneOf schemas, we need to preserve discriminator enum values
+        if (obj.discriminator && schema.properties) {
+          const propName = obj.discriminator.propertyName;
+          if (schema.properties[propName]) {
+            if (!extracted.properties) extracted.properties = {};
+            extracted.properties[propName] = {
+              enum:
+                schema.properties[propName].nts2Enum ||
+                schema.properties[propName].enum,
+            };
+          }
+        }
+        // Also preserve base enum values for all other properties in the extension
+        if (schema.properties) {
+          if (!extracted.properties) extracted.properties = {};
+          Object.keys(schema.properties).forEach((propName) => {
+            const prop = schema.properties[propName];
+            if (prop.enum) {
+              if (!extracted.properties[propName]) {
+                extracted.properties[propName] = {
+                  enum: prop.enum,
+                };
+              } else if (!extracted.properties[propName].enum) {
+                // If the property exists but doesn't have enum, add it
+                extracted.properties[propName].enum = prop.enum;
+              }
+            }
+          });
+        }
+        return extracted;
+      } else {
+        // If this branch doesn't have nts2Ref, preserve the base discriminator enum
+        if (obj.discriminator && schema.properties) {
+          const propName = obj.discriminator.propertyName;
+          if (schema.properties[propName]) {
+            return {
+              properties: {
+                [propName]: {
+                  enum: schema.properties[propName].enum,
+                },
+              },
+            };
+          }
+        }
+        // If no discriminator, return empty object to preserve structure
+        return {};
+      }
+    });
+    result.oneOf = processedOneOf;
+
+    // --- NEW: If this is an extension property inside a oneOf, promote it to the parent overlay ---
+    // If parentKey is set, and this object has extension metadata, return a stub for the parent
+    if (
+      parentKey &&
+      (result.ntsRef || result.required || result.discriminator)
+    ) {
+      // This is an extension property inside a oneOf branch
+      // Return a stub for the parent property with metadata and oneOf structure
+      const parentStub = {};
+      Object.assign(parentStub, result);
+      return parentStub;
+    }
+  }
+
+  // Copy other JSON Schema keywords that might be present
+  [
+    "type",
+    "format",
+    "minimum",
+    "maximum",
+    "minLength",
+    "maxLength",
+    "minItems",
+    "maxItems",
+  ].forEach((keyword) => {
+    if (obj[keyword] !== undefined) {
+      result[keyword] = obj[keyword];
+    }
+  });
+
+  return result;
+}
+
+// Add required array at parent levels if needed
+function addRequiredArrays(overlay, schema) {
+  traverse(overlay).forEach(function (node) {
+    if (node && node.properties) {
+      const schemaPath = "/" + this.path.join("/");
+      try {
+        const schemaNode = jp.get(schema, schemaPath);
+        if (schemaNode && schemaNode.nts2Required) {
+          // Filter to only include properties that exist in this overlay
+          const overlayProps = Object.keys(node.properties);
+          const requiredProps = schemaNode.nts2Required.filter((prop) =>
+            overlayProps.includes(prop)
+          );
+          if (requiredProps.length > 0) {
+            node.required = requiredProps;
+          }
+        }
+      } catch (err) {
+        // Path doesn't exist in schema, skip
+      }
+    }
+  });
+  return overlay;
+}
+
+// Generate extension overlays
+Object.entries(extensionMappings).forEach(([code, config]) => {
+  console.log(`\nGenerating extension overlay: ${code} (${config.name})`);
+
+  // Extract the specific paths
+  let overlay = extractPathsWithNts2Ref(combinedSchema, config.paths);
+
+  // Add required arrays where needed
+  overlay = addRequiredArrays(overlay, combinedSchema);
+
+  // Add schema metadata
+  overlay.$schema = "http://json-schema.org/draft-07/schema#";
+  overlay.$id = `https://trust.propdata.org.uk/schemas/v3/overlays/extensions/${code}.json`;
+  overlay.$comment = config.description;
+
+  // Write the overlay file
+  const fileName = path.join(
+    __dirname,
+    `../schemas/v3/overlays/extensions/${code}.json`
+  );
+  fs.writeFileSync(fileName, JSON.stringify(overlay, null, 2));
+  console.log(`Extension overlay ${code} written to ${fileName}`);
+});
+
+console.log("\nExtension overlay extraction complete!");