@pdtf/schemas 3.4.1-6 → 3.4.1-8

package/src/tests/v3/caching.test.js

@@ -2,7 +2,11 @@ const {
   getSubschema,
   getSubschemaValidator,
   getCacheStats,
+  getDetailedCacheStats,
   clearSchemaCache,
+  warmupCache,
+  pruneCacheByAge,
+  setCacheMaxSize,
   isPathValid,
 } = require("../../../index");
 
@@ -314,4 +318,274 @@ describe("Schema Caching System", () => {
       expect(timeMs).toBeLessThan(1);
     });
   });
+
+  describe("Enhanced Cache Management", () => {
+    beforeEach(() => {
+      clearSchemaCache();
+    });
+
+    describe("Enhanced Statistics", () => {
+      test("should track hit/miss statistics", () => {
+        const stats1 = getCacheStats();
+        expect(stats1.hits).toBe(0);
+        expect(stats1.misses).toBe(0);
+        expect(stats1.totalQueries).toBe(0);
+        expect(stats1.hitRate).toBe(0);
+
+        // First call should be a miss
+        getSubschema(testPath, schemaId, overlays);
+        const stats2 = getCacheStats();
+        expect(stats2.misses).toBe(1);
+        expect(stats2.hits).toBe(0);
+        expect(stats2.totalQueries).toBe(1);
+        expect(stats2.hitRate).toBe(0);
+
+        // Second call should be a hit
+        getSubschema(testPath, schemaId, overlays);
+        const stats3 = getCacheStats();
+        expect(stats3.misses).toBe(1);
+        expect(stats3.hits).toBe(1);
+        expect(stats3.totalQueries).toBe(2);
+        expect(stats3.hitRate).toBe(50);
+      });
+
+      test("should provide detailed cache statistics", () => {
+        // Add some entries using safe paths
+        getSubschema("/participants", schemaId, []);
+        getSubschema("/status", schemaId, []);
+
+        const detailedStats = getDetailedCacheStats();
+        
+        expect(detailedStats).toHaveProperty("totalEntries");
+        expect(detailedStats).toHaveProperty("hits");
+        expect(detailedStats).toHaveProperty("misses");
+        expect(detailedStats).toHaveProperty("hitRate");
+        expect(detailedStats).toHaveProperty("entries");
+        expect(detailedStats).toHaveProperty("oldestEntry");
+        expect(detailedStats).toHaveProperty("newestEntry");
+        
+        expect(detailedStats.entries).toHaveLength(2);
+        expect(detailedStats.entries[0]).toHaveProperty("key");
+        expect(detailedStats.entries[0]).toHaveProperty("createdAt");
+        expect(detailedStats.entries[0]).toHaveProperty("age");
+        expect(detailedStats.entries[0]).toHaveProperty("hasValidator");
+        expect(detailedStats.entries[0]).toHaveProperty("hasSubSchema");
+      });
+
+      test("should track memory usage estimates", () => {
+        getSubschema(testPath, schemaId, overlays);
+        
+        const stats = getCacheStats();
+        expect(stats.memoryUsage).toHaveProperty("entriesCount");
+        expect(stats.memoryUsage).toHaveProperty("estimatedSizeKB");
+        expect(stats.memoryUsage.entriesCount).toBe(1);
+        expect(typeof stats.memoryUsage.estimatedSizeKB).toBe("number");
+      });
+    });
+
+    describe("Cache Warming", () => {
+      test("should warm cache with default paths and overlays", () => {
+        const warmupStats = warmupCache();
+        
+        expect(warmupStats).toHaveProperty("totalAttempted");
+        expect(warmupStats).toHaveProperty("successful");
+        expect(warmupStats).toHaveProperty("failed");
+        expect(warmupStats).toHaveProperty("errors");
+        
+        expect(warmupStats.totalAttempted).toBeGreaterThan(0);
+        expect(warmupStats.successful).toBeGreaterThan(0);
+        expect(warmupStats.totalAttempted).toBe(warmupStats.successful + warmupStats.failed);
+
+        // Cache should now have entries
+        const stats = getCacheStats();
+        expect(stats.totalEntries).toBeGreaterThan(0);
+      });
+
+      test("should warm cache with custom paths and overlays", () => {
+        const customPaths = ["/propertyPack", "/participants"];
+        const customOverlays = [[], ["baspiV5"]];
+        
+        const warmupStats = warmupCache(customPaths, schemaId, customOverlays);
+        
+        expect(warmupStats.totalAttempted).toBe(customPaths.length * customOverlays.length);
+        expect(warmupStats.successful).toBeGreaterThan(0);
+
+        const stats = getCacheStats();
+        expect(stats.totalEntries).toBe(warmupStats.successful);
+      });
+
+      test("should handle invalid paths gracefully during warmup", () => {
+        const pathsWithInvalid = ["/participants", "/invalid/path", "/status"];
+        const warmupStats = warmupCache(pathsWithInvalid, schemaId, [[]]);
+        
+        expect(warmupStats.totalAttempted).toBe(3);
+        expect(warmupStats.successful).toBe(3); // All paths are processed, even invalid ones
+        expect(warmupStats.failed).toBe(0); // Invalid paths still "succeed" but create null schemas
+      });
+    });
+
+    describe("Selective Cache Clearing", () => {
+      test("should clear cache entries by pattern", () => {
+        // Populate cache with different patterns using safe paths
+        getSubschema("/participants", schemaId, []);
+        getSubschema("/status", schemaId, []);
+        getSubschema("/participants", schemaId, ["baspiV5"]);
+        
+        const stats1 = getCacheStats();
+        expect(stats1.totalEntries).toBe(3);
+        
+        // Clear entries containing "participants"
+        const cleared = clearSchemaCache("participants");
+        expect(cleared).toBe(2);
+        
+        const stats2 = getCacheStats();
+        expect(stats2.totalEntries).toBe(1);
+        
+        // Remaining entry should be status
+        expect(stats2.cacheKeys[0]).toContain("status");
+      });
+
+      test("should return number of entries cleared", () => {
+        getSubschema("/participants", schemaId, []);
+        getSubschema("/status", schemaId, []);
+        
+        const cleared = clearSchemaCache();
+        expect(cleared).toBe(2);
+        
+        const stats = getCacheStats();
+        expect(stats.totalEntries).toBe(0);
+      });
+
+      test("should reset statistics when clearing all cache", () => {
+        getSubschema(testPath, schemaId, overlays);
+        getSubschema(testPath, schemaId, overlays); // Hit
+        
+        const statsBefore = getCacheStats();
+        expect(statsBefore.hits).toBe(1);
+        expect(statsBefore.misses).toBe(1);
+        
+        clearSchemaCache();
+        
+        const statsAfter = getCacheStats();
+        expect(statsAfter.hits).toBe(0);
+        expect(statsAfter.misses).toBe(0);
+        expect(statsAfter.totalQueries).toBe(0);
+      });
+    });
+
+    describe("Cache Pruning", () => {
+      test("should prune entries by age", async () => {
+        // Add some entries using safe paths
+        getSubschema("/participants", schemaId, []);
+        
+        // Wait a bit
+        await new Promise(resolve => setTimeout(resolve, 10));
+        
+        getSubschema("/status", schemaId, []);
+        
+        const stats1 = getCacheStats();
+        expect(stats1.totalEntries).toBe(2);
+        
+        // Prune entries older than 5ms
+        const pruned = pruneCacheByAge(5);
+        expect(pruned).toBe(1); // Should remove the older entry
+        
+        const stats2 = getCacheStats();
+        expect(stats2.totalEntries).toBe(1);
+      });
+
+      test("should not prune entries younger than max age", () => {
+        getSubschema("/participants", schemaId, []);
+        
+        const pruned = pruneCacheByAge(60000); // 1 minute
+        expect(pruned).toBe(0);
+        
+        const stats = getCacheStats();
+        expect(stats.totalEntries).toBe(1);
+      });
+    });
+
+    describe("Cache Size Management", () => {
+      test("should limit cache size by removing oldest entries", () => {
+        // Add multiple entries using safe paths
+        getSubschema("/participants", schemaId, []);
+        getSubschema("/status", schemaId, []);
+        getSubschema("/participants", schemaId, ["baspiV5"]);
+        getSubschema("/status", schemaId, ["baspiV5"]);
+        
+        const stats1 = getCacheStats();
+        expect(stats1.totalEntries).toBe(4);
+        
+        // Limit to 2 entries
+        const removed = setCacheMaxSize(2);
+        expect(removed).toBe(2);
+        
+        const stats2 = getCacheStats();
+        expect(stats2.totalEntries).toBe(2);
+      });
+
+      test("should not remove entries if cache is under limit", () => {
+        getSubschema("/participants", schemaId, []);
+        
+        const removed = setCacheMaxSize(10);
+        expect(removed).toBe(0);
+        
+        const stats = getCacheStats();
+        expect(stats.totalEntries).toBe(1);
+      });
+
+      test("should remove oldest entries first", async () => {
+        // Add entries with delays to ensure different timestamps
+        getSubschema("/participants", schemaId, []);
+        await new Promise(resolve => setTimeout(resolve, 5));
+        
+        getSubschema("/status", schemaId, []);
+        await new Promise(resolve => setTimeout(resolve, 5));
+        
+        getSubschema("/participants", schemaId, ["baspiV5"]);
+        
+        const stats1 = getCacheStats();
+        expect(stats1.totalEntries).toBe(3);
+        
+        // Limit to 1 entry - should keep the newest one
+        setCacheMaxSize(1);
+        
+        const stats2 = getCacheStats();
+        expect(stats2.totalEntries).toBe(1);
+        expect(stats2.cacheKeys[0]).toContain("baspiV5");
+      });
+    });
+
+    describe("Integration with Existing Cache Behavior", () => {
+      test("enhanced stats should work with existing cache operations", () => {
+        // Use existing functions with a safe path
+        const safePath = "/participants";
+        const validator = getSubschemaValidator(safePath, schemaId, []);
+        const schema = getSubschema(safePath, schemaId, []);
+        const valid = isPathValid(safePath, schemaId, []);
+        
+        expect(typeof validator).toBe("function");
+        expect(schema).toBeDefined();
+        expect(valid).toBe(true);
+        
+        const stats = getCacheStats();
+        expect(stats.totalQueries).toBeGreaterThan(0);
+        expect(stats.hits).toBeGreaterThan(0);
+        expect(stats.totalEntries).toBe(1);
+      });
+
+      test("should maintain backward compatibility", () => {
+        // Existing test patterns should still work
+        const schema1 = getSubschema(testPath, schemaId, overlays);
+        const schema2 = getSubschema(testPath, schemaId, overlays);
+        
+        expect(schema1).toBe(schema2);
+        
+        const stats = getCacheStats();
+        expect(stats).toHaveProperty("totalEntries");
+        expect(stats).toHaveProperty("cacheKeys");
+        expect(stats.totalEntries).toBe(1);
+      });
+    });
+  });
 });

package/src/tests/v3/transactionSchema.test.js

@@ -449,14 +449,61 @@ test("validates a valid contract", () => {
   expect(isValid).toBe(true);
 });
 
-test("waterAndDrainage state with mainsFoulDrainage No missing offMainsDrainageSystem is now invalid under nts2023 overlay", () => {
+test("ntsl2025 requires parking and listingAndConservation fields", () => {
+  // This test validates that the ntsl2025 overlay correctly enforces required fields
+  // for the parking and listingAndConservation sections.
+  // It should fail validation when these fields are missing.
+  // NOTE: This test will fail until overlays are regenerated from combined.json
+  const validator = getValidator(schemaId, ["ntsl2025"]);
+  const clonedExampleTransaction = JSON.parse(
+    JSON.stringify(exampleTransaction)
+  );
+  
+  // Set up lettings-specific fields
+  clonedExampleTransaction.propertyPack.lettingInformation = {
+    rent: 3500,
+    rentFrequency: "Monthly",
+    securityDeposit: 5000,
+  };
+  delete clonedExampleTransaction.propertyPack.priceInformation;
+  delete clonedExampleTransaction.propertyPack.ownership;
+  
+  // Remove ALL parking fields - this should make validation fail
+  delete clonedExampleTransaction.propertyPack.parking.parkingArrangements;
+  delete clonedExampleTransaction.propertyPack.parking.disabledParking;
+  delete clonedExampleTransaction.propertyPack.parking.controlledParking;
+  delete clonedExampleTransaction.propertyPack.parking.electricVehicleChargingPoint;
+  
+  // Remove ALL listingAndConservation fields - this should make validation fail
+  delete clonedExampleTransaction.propertyPack.listingAndConservation.isListed;
+  delete clonedExampleTransaction.propertyPack.listingAndConservation.isConservationArea;
+  delete clonedExampleTransaction.propertyPack.listingAndConservation.hasTreePreservationOrder;
+  
+  const isValid = validator(clonedExampleTransaction);
+  
+  // The data should be INVALID because required fields are missing
+  expect(isValid).toBe(false);
+  
+  // Specifically check that parking and listingAndConservation fields are reported as missing
+  const errorMessages = validator.errors.map(e => e.message);
+  expect(errorMessages).toContain("must have required property 'parkingArrangements'");
+  expect(errorMessages).toContain("must have required property 'disabledParking'");
+  expect(errorMessages).toContain("must have required property 'controlledParking'");
+  expect(errorMessages).toContain("must have required property 'electricVehicleChargingPoint'");
+  expect(errorMessages).toContain("must have required property 'isListed'");
+  expect(errorMessages).toContain("must have required property 'isConservationArea'");
+  expect(errorMessages).toContain("must have required property 'hasTreePreservationOrder'");
+});
+
+test("waterAndDrainage state is invalid under nts2023 overlay when mainsFoulDrainage yesNo is Not known", () => {
   const validator = getValidator(schemaId, ["nts2023"]);
   const clonedExampleTransaction = JSON.parse(
     JSON.stringify(exampleTransaction)
   );
   
-  // Set the waterAndDrainage to the state from the user request
-  // This SHOULD be invalid because offMainsDrainageSystem is required when mainsFoulDrainage.yesNo is "No"
+  // Set the waterAndDrainage to an actually invalid state for nts2023
+  // The nts overlay restricts mainsFoulDrainage.yesNo to only "Yes" or "No" 
+  // (not "Not known" like the base schema allows)
   clonedExampleTransaction.propertyPack.waterAndDrainage = {
     "water": {
       "mainsWater": {
@@ -471,8 +518,7 @@ test("waterAndDrainage state with mainsFoulDrainage No missing offMainsDrainageS
         "yesNo": "Yes"
       },
       "mainsFoulDrainage": {
-        "yesNo": "No"
-        // Missing offMainsDrainageSystem - this should make validation fail
+        "yesNo": "Not known"  // This is invalid under nts2023 overlay
       }
     }
   };
@@ -480,23 +526,22 @@ test("waterAndDrainage state with mainsFoulDrainage No missing offMainsDrainageS
   const isValid = validator(clonedExampleTransaction);
   expect(isValid).toBe(false);
   
-  // Check that the validation error is about missing offMainsDrainageSystem
+  // Check that the validation error is about the invalid enum value
   const relevantError = validator.errors.find(error => 
     error.instancePath.includes('mainsFoulDrainage') && 
-    error.message.includes('offMainsDrainageSystem')
+    error.message.includes('must be equal to one of the allowed values')
   );
   expect(relevantError).toBeDefined();
 });
 
-test("waterAndDrainage state is invalid under nts2023 overlay when mainsFoulDrainage yesNo is Not known", () => {
+test("waterAndDrainage state with mainsFoulDrainage No missing offMainsDrainageSystem is now invalid under nts2023 overlay", () => {
   const validator = getValidator(schemaId, ["nts2023"]);
   const clonedExampleTransaction = JSON.parse(
     JSON.stringify(exampleTransaction)
   );
   
-  // Set the waterAndDrainage to an actually invalid state for nts2023
-  // The nts overlay restricts mainsFoulDrainage.yesNo to only "Yes" or "No" 
-  // (not "Not known" like the base schema allows)
+  // Set the waterAndDrainage to the state from the user request
+  // This SHOULD be invalid because offMainsDrainageSystem is required when mainsFoulDrainage.yesNo is "No"
   clonedExampleTransaction.propertyPack.waterAndDrainage = {
     "water": {
       "mainsWater": {
@@ -511,7 +556,8 @@ test("waterAndDrainage state is invalid under nts2023 overlay when mainsFoulDrai
         "yesNo": "Yes"
       },
       "mainsFoulDrainage": {
-        "yesNo": "Not known"  // This is invalid under nts2023 overlay
+        "yesNo": "No"
+        // Missing offMainsDrainageSystem - this should make validation fail
       }
     }
   };
@@ -519,10 +565,10 @@ test("waterAndDrainage state is invalid under nts2023 overlay when mainsFoulDrai
   const isValid = validator(clonedExampleTransaction);
   expect(isValid).toBe(false);
   
-  // Check that the validation error is about the invalid enum value
+  // Check that the validation error is about missing offMainsDrainageSystem
   const relevantError = validator.errors.find(error => 
     error.instancePath.includes('mainsFoulDrainage') && 
-    error.message.includes('must be equal to one of the allowed values')
+    error.message.includes('offMainsDrainageSystem')
   );
   expect(relevantError).toBeDefined();
 });

package/src/utils/compactSkeleton.js

@@ -0,0 +1,84 @@
+const fs = require("fs");
+
+// Read the current skeleton
+const skeleton = require("../schemas/v3/skeleton.json");
+
+// Convert to ultra-compact format
+const toCompact = (obj, indent = "") => {
+  if (typeof obj === "string") {
+    // Type indicators
+    if (["string", "number", "integer", "boolean", "variant"].includes(obj)) {
+      return obj[0]; // Just first letter: s, n, i, b, v
+    }
+    return obj;
+  }
+  
+  if (Array.isArray(obj)) {
+    if (obj.length === 0) return "[]";
+    // Array with content
+    const inner = toCompact(obj[0], indent + "  ");
+    if (typeof inner === "string" && inner.length === 1) {
+      return `[${inner}]`; // Simple array like [s]
+    }
+    return `[\n${indent}  ${inner}\n${indent}]`;
+  }
+  
+  if (obj && typeof obj === "object") {
+    const keys = Object.keys(obj);
+    if (keys.length === 0) return "{}";
+    
+    // Check if it's a simple object with only type indicators
+    const allSimple = keys.every(k => 
+      typeof obj[k] === "string" && 
+      ["s", "n", "i", "b", "v"].includes(toCompact(obj[k]))
+    );
+    
+    if (allSimple && keys.length < 5) {
+      // Inline format: {a:s b:n c:s}
+      return `{${keys.map(k => `${k}:${toCompact(obj[k])}`).join(" ")}}`;
+    }
+    
+    // Multi-line format
+    const lines = keys.map(k => {
+      const val = toCompact(obj[k], indent + "  ");
+      if (typeof val === "string" && val.length === 1) {
+        return `${indent}  ${k}:${val}`;
+      } else if (typeof val === "string" && (val === "{}" || val === "[]")) {
+        return `${indent}  ${k}${val}`;
+      } else if (typeof val === "string" && val.startsWith("{") && val.endsWith("}")) {
+        return `${indent}  ${k}${val}`;
+      } else if (typeof val === "string" && val.startsWith("[") && !val.includes("\n")) {
+        return `${indent}  ${k}${val}`;
+      } else {
+        return `${indent}  ${k}:\n${indent}  ${val}`;
+      }
+    });
+    
+    return `{\n${lines.join("\n")}\n${indent}}`;
+  }
+  
+  return String(obj);
+};
+
+// Generate ultra-compact version
+const compact = toCompact(skeleton);
+
+// Write compact version
+fs.writeFileSync("../schemas/v3/compactSkeleton.txt", compact);
+
+console.log("Compact skeleton written to ../schemas/v3/compactSkeleton.txt");
+console.log(`Size: ${compact.length} bytes`);
+
+// Count tokens
+try {
+  const GPT3Encoder = require('gpt-3-encoder');
+  const encoded = GPT3Encoder.encode(compact);
+  console.log(`Tokens (GPT-3): ${encoded.length}`);
+  console.log(`Reduction: ${((1 - encoded.length / 78734) * 100).toFixed(1)}% fewer tokens than JSON skeleton`);
+  
+  // Show sample
+  console.log("\nFirst 500 chars:");
+  console.log(compact.substring(0, 500));
+} catch (e) {
+  console.log(`Estimated tokens: ~${Math.ceil(compact.length / 4)}`);
+}

package/src/utils/countTokens.js

@@ -0,0 +1,67 @@
+const fs = require('fs');
+
+// Try to use tiktoken or a similar library
+// For Claude/Anthropic models, we can estimate using the cl100k_base encoding
+// which is similar to what Claude uses
+
+// First, let's check what's available
+try {
+  // Option 1: Use gpt-tokenizer (common npm package)
+  const { encode } = require('gpt-tokenizer');
+  
+  const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+  const tokens = encode(content);
+  
+  console.log(`Using gpt-tokenizer:`);
+  console.log(`Total tokens: ${tokens.length}`);
+  console.log(`File size: ${content.length} characters`);
+  console.log(`Ratio: ${(content.length / tokens.length).toFixed(2)} characters per token`);
+  
+} catch (e1) {
+  console.log("gpt-tokenizer not found, trying gpt-3-encoder...");
+  
+  try {
+    // Option 2: Use gpt-3-encoder
+    const GPT3Encoder = require('gpt-3-encoder');
+    
+    const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+    const encoded = GPT3Encoder.encode(content);
+    
+    console.log(`Using gpt-3-encoder:`);
+    console.log(`Total tokens: ${encoded.length}`);
+    console.log(`File size: ${content.length} characters`);
+    console.log(`Ratio: ${(content.length / encoded.length).toFixed(2)} characters per token`);
+    
+  } catch (e2) {
+    console.log("No tokenizer library found. Falling back to estimation...");
+    
+    // Option 3: Manual estimation
+    const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+    
+    // Count different types of content for better estimation
+    const lines = content.split('\n');
+    const words = content.split(/\s+/);
+    const punctuation = content.match(/[{}:,\[\]"]/g) || [];
+    
+    // Rough estimation:
+    // - JSON structure tokens (brackets, colons, quotes): ~1 token each
+    // - Words: ~1 token each
+    // - Whitespace: usually absorbed into adjacent tokens
+    
+    const structureTokens = punctuation.length;
+    const wordTokens = words.length;
+    const estimatedTokens = Math.ceil((structureTokens + wordTokens) * 0.75); // 0.75 factor for overlap
+    
+    console.log("Manual estimation:");
+    console.log(`File size: ${content.length} characters`);
+    console.log(`Lines: ${lines.length}`);
+    console.log(`Words: ${words.length}`);
+    console.log(`JSON punctuation: ${punctuation.length}`);
+    console.log(`Estimated tokens: ~${estimatedTokens}`);
+    console.log(`Ratio: ${(content.length / estimatedTokens).toFixed(2)} characters per token`);
+    
+    // Also do a simple character-based estimation
+    const simpleEstimate = Math.ceil(content.length / 4);
+    console.log(`\nSimple estimation (chars/4): ~${simpleEstimate} tokens`);
+  }
+}