@pdtf/schemas 3.4.1-7 → 3.5.0

package/src/utils/compactSkeleton.js

@@ -0,0 +1,84 @@
+const fs = require("fs");
+
+// Read the current skeleton
+const skeleton = require("../schemas/v3/skeleton.json");
+
+// Convert to ultra-compact format
+const toCompact = (obj, indent = "") => {
+  if (typeof obj === "string") {
+    // Type indicators
+    if (["string", "number", "integer", "boolean", "variant"].includes(obj)) {
+      return obj[0]; // Just first letter: s, n, i, b, v
+    }
+    return obj;
+  }
+  
+  if (Array.isArray(obj)) {
+    if (obj.length === 0) return "[]";
+    // Array with content
+    const inner = toCompact(obj[0], indent + "  ");
+    if (typeof inner === "string" && inner.length === 1) {
+      return `[${inner}]`; // Simple array like [s]
+    }
+    return `[\n${indent}  ${inner}\n${indent}]`;
+  }
+  
+  if (obj && typeof obj === "object") {
+    const keys = Object.keys(obj);
+    if (keys.length === 0) return "{}";
+    
+    // Check if it's a simple object with only type indicators
+    const allSimple = keys.every(k => 
+      typeof obj[k] === "string" && 
+      ["s", "n", "i", "b", "v"].includes(toCompact(obj[k]))
+    );
+    
+    if (allSimple && keys.length < 5) {
+      // Inline format: {a:s b:n c:s}
+      return `{${keys.map(k => `${k}:${toCompact(obj[k])}`).join(" ")}}`;
+    }
+    
+    // Multi-line format
+    const lines = keys.map(k => {
+      const val = toCompact(obj[k], indent + "  ");
+      if (typeof val === "string" && val.length === 1) {
+        return `${indent}  ${k}:${val}`;
+      } else if (typeof val === "string" && (val === "{}" || val === "[]")) {
+        return `${indent}  ${k}${val}`;
+      } else if (typeof val === "string" && val.startsWith("{") && val.endsWith("}")) {
+        return `${indent}  ${k}${val}`;
+      } else if (typeof val === "string" && val.startsWith("[") && !val.includes("\n")) {
+        return `${indent}  ${k}${val}`;
+      } else {
+        return `${indent}  ${k}:\n${indent}  ${val}`;
+      }
+    });
+    
+    return `{\n${lines.join("\n")}\n${indent}}`;
+  }
+  
+  return String(obj);
+};
+
+// Generate ultra-compact version
+const compact = toCompact(skeleton);
+
+// Write compact version
+fs.writeFileSync("../schemas/v3/compactSkeleton.txt", compact);
+
+console.log("Compact skeleton written to ../schemas/v3/compactSkeleton.txt");
+console.log(`Size: ${compact.length} bytes`);
+
+// Count tokens
+try {
+  const GPT3Encoder = require('gpt-3-encoder');
+  const encoded = GPT3Encoder.encode(compact);
+  console.log(`Tokens (GPT-3): ${encoded.length}`);
+  console.log(`Reduction: ${((1 - encoded.length / 78734) * 100).toFixed(1)}% fewer tokens than JSON skeleton`);
+  
+  // Show sample
+  console.log("\nFirst 500 chars:");
+  console.log(compact.substring(0, 500));
+} catch (e) {
+  console.log(`Estimated tokens: ~${Math.ceil(compact.length / 4)}`);
+}

package/src/utils/countTokens.js

@@ -0,0 +1,67 @@
+const fs = require('fs');
+
+// Try to use tiktoken or a similar library
+// For Claude/Anthropic models, we can estimate using the cl100k_base encoding
+// which is similar to what Claude uses
+
+// First, let's check what's available
+try {
+  // Option 1: Use gpt-tokenizer (common npm package)
+  const { encode } = require('gpt-tokenizer');
+  
+  const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+  const tokens = encode(content);
+  
+  console.log(`Using gpt-tokenizer:`);
+  console.log(`Total tokens: ${tokens.length}`);
+  console.log(`File size: ${content.length} characters`);
+  console.log(`Ratio: ${(content.length / tokens.length).toFixed(2)} characters per token`);
+  
+} catch (e1) {
+  console.log("gpt-tokenizer not found, trying gpt-3-encoder...");
+  
+  try {
+    // Option 2: Use gpt-3-encoder
+    const GPT3Encoder = require('gpt-3-encoder');
+    
+    const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+    const encoded = GPT3Encoder.encode(content);
+    
+    console.log(`Using gpt-3-encoder:`);
+    console.log(`Total tokens: ${encoded.length}`);
+    console.log(`File size: ${content.length} characters`);
+    console.log(`Ratio: ${(content.length / encoded.length).toFixed(2)} characters per token`);
+    
+  } catch (e2) {
+    console.log("No tokenizer library found. Falling back to estimation...");
+    
+    // Option 3: Manual estimation
+    const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+    
+    // Count different types of content for better estimation
+    const lines = content.split('\n');
+    const words = content.split(/\s+/);
+    const punctuation = content.match(/[{}:,\[\]"]/g) || [];
+    
+    // Rough estimation:
+    // - JSON structure tokens (brackets, colons, quotes): ~1 token each
+    // - Words: ~1 token each
+    // - Whitespace: usually absorbed into adjacent tokens
+    
+    const structureTokens = punctuation.length;
+    const wordTokens = words.length;
+    const estimatedTokens = Math.ceil((structureTokens + wordTokens) * 0.75); // 0.75 factor for overlap
+    
+    console.log("Manual estimation:");
+    console.log(`File size: ${content.length} characters`);
+    console.log(`Lines: ${lines.length}`);
+    console.log(`Words: ${words.length}`);
+    console.log(`JSON punctuation: ${punctuation.length}`);
+    console.log(`Estimated tokens: ~${estimatedTokens}`);
+    console.log(`Ratio: ${(content.length / estimatedTokens).toFixed(2)} characters per token`);
+    
+    // Also do a simple character-based estimation
+    const simpleEstimate = Math.ceil(content.length / 4);
+    console.log(`\nSimple estimation (chars/4): ~${simpleEstimate} tokens`);
+  }
+}

package/src/utils/extractOverlay.js

@@ -29,25 +29,58 @@ const extractFields = [
 
 const flattenSkeleton = (schema) => {
   if (!schema) return undefined;
+  
+  // Handle arrays - mark with special notation
+  if (schema.type === "array" && schema.items) {
+    const itemSchema = flattenSkeleton(schema.items);
+    // Use array notation to indicate this is an array
+    return [itemSchema];
+  }
+  
+  // Handle primitives - return type indicator
+  if (schema.type && !schema.properties && !schema.oneOf && !schema.items) {
+    // Return a type indicator for primitive types
+    if (schema.type === "string") return "string";
+    if (schema.type === "number") return "number";
+    if (schema.type === "integer") return "integer";
+    if (schema.type === "boolean") return "boolean";
+    if (schema.type === "null") return "null";
+  }
+  
   let returnStructure = {};
+  
+  // Handle object properties
   if (schema.properties) {
     Object.keys(schema.properties).forEach((key) => {
       returnStructure[key] = flattenSkeleton(schema.properties[key]);
     });
   }
+  
+  // Handle oneOf - merge all possible properties
   if (schema.oneOf) {
     schema.oneOf.forEach((aOneOf) => {
       if (aOneOf.properties) {
         Object.entries(aOneOf.properties).forEach(([key, value]) => {
-          returnStructure[key] = flattenSkeleton(value);
+          // If property already exists and differs, mark as variant
+          const newValue = flattenSkeleton(value);
+          if (returnStructure[key] && JSON.stringify(returnStructure[key]) !== JSON.stringify(newValue)) {
+            // For primitive types, just mark as variant type
+            if (typeof returnStructure[key] === "string" || typeof newValue === "string") {
+              returnStructure[key] = "variant";
+            } else {
+              // For complex types, merge properties
+              returnStructure[key] = { ...returnStructure[key], ...newValue, _variants: true };
+            }
+          } else {
+            returnStructure[key] = newValue;
+          }
         });
       }
     });
   }
-  if (schema.items) {
-    returnStructure = flattenSkeleton(schema.items);
-  }
-  return returnStructure;
+  
+  // Return empty object for objects without properties
+  return Object.keys(returnStructure).length > 0 ? returnStructure : {};
 };
 
 const extractOverlay = (sourceSchema, ref) => {
@@ -197,3 +230,61 @@ fs.writeFileSync(
   JSON.stringify(skeletonSchemaFlattened, null, 2)
 );
 console.log("Flat Skeleton schema written to ../schemas/v3/skeleton.json");
+
+// Generate compact skeleton format for better token efficiency
+const toCompact = (obj, indent = "") => {
+  if (typeof obj === "string") {
+    // Skip type indicators entirely
+    return "";
+  }
+  
+  if (Array.isArray(obj)) {
+    if (obj.length === 0) return "[]";
+    // Array with content
+    const inner = toCompact(obj[0], indent);
+    if (inner === "") {
+      return "[]"; // Array of primitives
+    }
+    return `[\n${inner}\n${indent}]`;
+  }
+  
+  if (obj && typeof obj === "object") {
+    const keys = Object.keys(obj);
+    if (keys.length === 0) return "";
+    
+    // Process each key
+    const lines = [];
+    keys.forEach(k => {
+      const val = toCompact(obj[k], indent + "  ");
+      
+      if (val === "") {
+        // Leaf node - just show the property name
+        lines.push(`${indent}  ${k}`);
+      } else if (val === "[]") {
+        // Array property
+        lines.push(`${indent}  ${k}[]`);
+      } else if (val.startsWith("[")) {
+        // Array with nested content - simple concatenation
+        lines.push(`${indent}  ${k}${val}`);
+      } else {
+        // Object property - show name on its own line, content below
+        lines.push(`${indent}  ${k}`);
+        lines.push(val);
+      }
+    });
+    
+    return lines.length > 0 ? lines.join("\n") : "";
+  }
+  
+  return "";
+};
+
+// Since toCompact returns lines without wrapping braces, add them for the root
+const innerContent = toCompact(skeletonSchemaFlattened, "");
+const compactSkeleton = innerContent ? innerContent : "";
+
+fs.writeFileSync(
+  "../schemas/v3/compactSkeleton.txt",
+  compactSkeleton
+);
+console.log("Compact Skeleton written to ../schemas/v3/compactSkeleton.txt");

package/src/utils/minimalSkeleton.js

@@ -0,0 +1,85 @@
+const fs = require("fs");
+
+// Read the current skeleton
+const skeleton = require("../schemas/v3/skeleton.json");
+
+// Ultra-minimal format: use indentation only, no brackets
+const toMinimal = (obj, prefix = "", isArrayItem = false) => {
+  const lines = [];
+  
+  if (typeof obj === "string") {
+    // Type indicators - single letter
+    const typeMap = {
+      "string": "s",
+      "number": "n", 
+      "integer": "i",
+      "boolean": "b",
+      "variant": "v"
+    };
+    return typeMap[obj] || obj;
+  }
+  
+  if (Array.isArray(obj)) {
+    lines.push(prefix + "[]");
+    if (obj.length > 0) {
+      const itemLines = toMinimal(obj[0], prefix + "  ", true);
+      lines.push(...(Array.isArray(itemLines) ? itemLines : [itemLines]));
+    }
+    return lines;
+  }
+  
+  if (obj && typeof obj === "object") {
+    const keys = Object.keys(obj);
+    if (keys.length === 0) {
+      return prefix + "{}";
+    }
+    
+    keys.forEach((key, index) => {
+      const val = obj[key];
+      const keyPrefix = prefix + key;
+      
+      if (typeof val === "string") {
+        const type = toMinimal(val);
+        lines.push(keyPrefix + ":" + type);
+      } else if (Array.isArray(val)) {
+        const arrLines = toMinimal(val, keyPrefix);
+        lines.push(...arrLines);
+      } else if (val && typeof val === "object" && Object.keys(val).length > 0) {
+        lines.push(keyPrefix);
+        const subLines = toMinimal(val, prefix + "  ");
+        lines.push(...(Array.isArray(subLines) ? subLines : [subLines]));
+      } else {
+        lines.push(keyPrefix);
+      }
+    });
+    
+    return lines;
+  }
+  
+  return prefix + String(obj);
+};
+
+// Generate minimal version
+const minimalLines = toMinimal(skeleton);
+const minimal = minimalLines.join("\n");
+
+// Write minimal version
+fs.writeFileSync("../schemas/v3/minimalSkeleton.txt", minimal);
+
+console.log("Minimal skeleton written to ../schemas/v3/minimalSkeleton.txt");
+console.log(`Size: ${minimal.length} bytes`);
+console.log(`Lines: ${minimalLines.length}`);
+
+// Count tokens
+try {
+  const GPT3Encoder = require('gpt-3-encoder');
+  const encoded = GPT3Encoder.encode(minimal);
+  console.log(`Tokens (GPT-3): ${encoded.length}`);
+  console.log(`Reduction: ${((1 - encoded.length / 78734) * 100).toFixed(1)}% fewer tokens than JSON skeleton`);
+  
+  // Show sample
+  console.log("\nFirst 20 lines:");
+  console.log(minimalLines.slice(0, 20).join("\n"));
+} catch (e) {
+  console.log(`Estimated tokens: ~${Math.ceil(minimal.length / 4)}`);
+}

package/src/utils/pathSkeleton.js

@@ -0,0 +1,181 @@
+const fs = require("fs");
+const traverse = require("traverse");
+
+const combinedSchema = require("../schemas/v3/combined.json");
+
+// Extract fields that have overlay-specific properties
+const extractFields = [
+  "baspi4", "baspi5", "nts", "nts2", "ntsl", "ntsl2",
+  "ta6", "ta7", "ta10", "lpe1", "fme1", "piq",
+  "con29R", "con29DW", "llc1", "rds", "oc1", "sr24",
+];
+
+// Delete properties we don't want in the skeleton
+const deleteProperties = (sourceSchema, propertyNames) => {
+  traverse(sourceSchema).forEach(function (element) {
+    if (propertyNames.includes(this.key)) {
+      this.delete(true);
+    }
+  });
+  return sourceSchema;
+};
+
+// Collect all paths in path notation
+const collectPaths = (schema, currentPath = "") => {
+  const paths = [];
+  
+  if (!schema || typeof schema !== 'object') {
+    return paths;
+  }
+  
+  // Handle arrays
+  if (schema.type === "array" && schema.items) {
+    const arrayPath = currentPath + "[]";
+    paths.push(arrayPath);
+    // Continue with items
+    const itemPaths = collectPaths(schema.items, arrayPath);
+    paths.push(...itemPaths);
+    return paths;
+  }
+  
+  // Handle primitives - add type suffix
+  if (schema.type && !schema.properties && !schema.oneOf && !schema.items) {
+    let typeChar = "";
+    switch(schema.type) {
+      case "string": typeChar = ":s"; break;
+      case "number": typeChar = ":n"; break;
+      case "integer": typeChar = ":i"; break;
+      case "boolean": typeChar = ":b"; break;
+      default: typeChar = "";
+    }
+    if (currentPath) {
+      paths.push(currentPath + typeChar);
+    }
+    return paths;
+  }
+  
+  // Handle objects with properties
+  if (schema.properties) {
+    // If this is an object without a path yet, don't add it
+    if (currentPath && !schema.type?.includes("array")) {
+      paths.push(currentPath);
+    }
+    
+    Object.keys(schema.properties).forEach((propName) => {
+      const newPath = currentPath ? `${currentPath}.${propName}` : propName;
+      const propPaths = collectPaths(schema.properties[propName], newPath);
+      paths.push(...propPaths);
+    });
+  }
+  
+  // Handle oneOf - collect all possible properties
+  if (schema.oneOf) {
+    const variantProps = new Set();
+    schema.oneOf.forEach((oneOfSchema) => {
+      if (oneOfSchema.properties) {
+        Object.keys(oneOfSchema.properties).forEach((propName) => {
+          variantProps.add(propName);
+        });
+      }
+    });
+    
+    // Process each unique property
+    variantProps.forEach((propName) => {
+      const newPath = currentPath ? `${currentPath}.${propName}` : propName;
+      // Check if all variants have the same type
+      let commonType = null;
+      let isVariant = false;
+      
+      schema.oneOf.forEach((oneOfSchema) => {
+        if (oneOfSchema.properties && oneOfSchema.properties[propName]) {
+          const propType = oneOfSchema.properties[propName].type;
+          if (commonType === null) {
+            commonType = propType;
+          } else if (commonType !== propType) {
+            isVariant = true;
+          }
+        }
+      });
+      
+      if (isVariant || !commonType) {
+        paths.push(newPath + ":v"); // variant
+      } else {
+        // Process the first variant's version of this property
+        for (const oneOfSchema of schema.oneOf) {
+          if (oneOfSchema.properties && oneOfSchema.properties[propName]) {
+            const propPaths = collectPaths(oneOfSchema.properties[propName], newPath);
+            paths.push(...propPaths);
+            break;
+          }
+        }
+      }
+    });
+  }
+  
+  // Empty objects
+  if (currentPath && !schema.properties && !schema.oneOf && !schema.items && !schema.type) {
+    paths.push(currentPath);
+  }
+  
+  return paths;
+};
+
+// Remove duplicates
+const deduplicatePaths = (paths) => {
+  return [...new Set(paths)];
+};
+
+// Sort paths for readability
+const sortPaths = (paths) => {
+  return paths.sort((a, b) => {
+    // Sort by depth first (fewer dots = higher up)
+    const depthA = (a.match(/\./g) || []).length;
+    const depthB = (b.match(/\./g) || []).length;
+    if (depthA !== depthB) return depthA - depthB;
+    
+    // Then alphabetically
+    return a.localeCompare(b);
+  });
+};
+
+// Create clean schema
+const coreSchema = deleteProperties(JSON.parse(JSON.stringify(combinedSchema)), [
+  "discriminator",
+  ...extractFields.map((item) => `${item}Ref`),
+  ...extractFields.map((item) => `${item}Required`),
+  ...extractFields.map((item) => `${item}Title`),
+  ...extractFields.map((item) => `${item}Description`),
+  ...extractFields.map((item) => `${item}Enum`),
+]);
+
+// Remove metadata
+const skeletonSchema = deleteProperties(coreSchema, [
+  "$schema", "$id", "title", "description", "required",
+  "enum", "minItems", "minLength", "format", "minimum", "maximum",
+]);
+
+// Collect paths
+const allPaths = collectPaths(skeletonSchema);
+const uniquePaths = deduplicatePaths(allPaths);
+const sortedPaths = sortPaths(uniquePaths);
+
+// Write as text file
+const pathContent = sortedPaths.join('\n');
+fs.writeFileSync("../schemas/v3/pathSkeleton.txt", pathContent);
+
+// Also create a JSON version for comparison
+fs.writeFileSync("../schemas/v3/pathSkeleton.json", JSON.stringify(sortedPaths, null, 2));
+
+console.log(`Path skeleton written to ../schemas/v3/pathSkeleton.txt`);
+console.log(`Total paths: ${sortedPaths.length}`);
+console.log(`Text size: ${pathContent.length} bytes`);
+
+// Count tokens
+try {
+  const GPT3Encoder = require('gpt-3-encoder');
+  const encoded = GPT3Encoder.encode(pathContent);
+  console.log(`Tokens (GPT-3): ${encoded.length}`);
+  console.log(`Reduction: ${((1 - encoded.length / 78734) * 100).toFixed(1)}% fewer tokens than JSON skeleton`);
+} catch (e) {
+  console.log(`Estimated tokens: ~${Math.ceil(pathContent.length / 4)}`);
+}

package/src/schemas/v3/property-transaction-milestones.json

@@ -1,91 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "Transaction Milestones",
-  "description": "Schema for tracking transaction milestones throughout the conveyancing process",
-  "type": "object",
-  "properties": {
-    "listed": {
-      "type": "object",
-      "properties": {
-        "completed": {
-          "type": "string",
-          "format": "date-time"
-        }
-      }
-    },
-    "legalForms": {
-      "type": "object",
-      "properties": {
-        "completed": {
-          "type": "string",
-          "format": "date-time"
-        }
-      }
-    },
-    "soldSubjectToContract": {
-      "type": "object",
-      "properties": {
-        "completed": {
-          "type": "string",
-          "format": "date-time"
-        }
-      }
-    },
-    "searches": {
-      "type": "object",
-      "properties": {
-        "ordered": {
-          "type": "string",
-          "format": "date-time"
-        },
-        "expected": {
-          "type": "string",
-          "format": "date-time"
-        },
-        "completed": {
-          "type": "string",
-          "format": "date-time"
-        }
-      }
-    },
-    "enquiries": {
-      "type": "object",
-      "properties": {
-        "completed": {
-          "type": "string",
-          "format": "date-time"
-        }
-      }
-    },
-    "exchangeOfContracts": {
-      "type": "object",
-      "properties": {
-        "expected": {
-          "type": "string",
-          "format": "date-time"
-        },
-        "completed": {
-          "type": "string",
-          "format": "date-time"
-        }
-      }
-    },
-    "completion": {
-      "type": "object",
-      "properties": {
-        "started": {
-          "type": "string",
-          "format": "date-time"
-        },
-        "expected": {
-          "type": "string",
-          "format": "date-time"
-        },
-        "completed": {
-          "type": "string",
-          "format": "date-time"
-        }
-      }
-    }
-  }
-}