@pdtf/schemas 3.6.0-dev.3 → 3.6.0-dev.4

package/charming-napier-57a77c/src/tests/v3/verifiedClaimsValidator.test.js

@@ -0,0 +1,195 @@
+const { validateVerifiedClaims } = require("../../../index.js");
+
+const exampleVouch = require("../../examples/v3/exampleVouch.json");
+const exampleDocumentedVouch = require("../../examples/v3/exampleDocumentedVouch.json");
+const v3SchemaId =
+  "https://trust.propdata.org.uk/schemas/v3/pdtf-transaction.json";
+
+test("returns an empty array for a valid claim", () => {
+  expect(validateVerifiedClaims([exampleVouch])).toEqual([]);
+});
+
+test("returns an empty array for a valid array of claims", () => {
+  const claimsArray = [exampleVouch, exampleDocumentedVouch];
+  expect(validateVerifiedClaims(claimsArray)).toEqual([]);
+});
+
+test("returns an array with an error stating if path is incorrect", () => {
+  const clonedVouch = JSON.parse(JSON.stringify(exampleVouch));
+  const originalPath = Object.keys(clonedVouch.claims)[0];
+  const data = clonedVouch.claims[originalPath];
+  clonedVouch.claims = {
+    "/propertyPack/INVALID/councilTax": data,
+  };
+  // v3 schema, no overlay
+  expect(validateVerifiedClaims([clonedVouch], v3SchemaId, null)).toEqual([
+    "Path /propertyPack/INVALID/councilTax is not a valid PDTF schema path",
+  ]);
+});
+
+test("returns errors if BASPI requirements are not met and BASPI overlay is specified", () => {
+  const clonedVouch = JSON.parse(JSON.stringify(exampleVouch));
+  clonedVouch.claims = {
+    "/propertyPack/delayFactors": {
+      hasDelayFactors: { yesNo: "Yes" },
+    },
+  };
+  const errors = validateVerifiedClaims([clonedVouch], v3SchemaId, ["baspiV4"]);
+  expect(errors).toEqual([
+    {
+      instancePath: "/hasDelayFactors",
+      schemaPath: "#/properties/hasDelayFactors/oneOf/1/required",
+      keyword: "required",
+      params: { missingProperty: "details" },
+      message: "must have required property 'details'",
+    },
+    {
+      instancePath: "/hasDelayFactors",
+      schemaPath: "#/properties/hasDelayFactors/oneOf/1/required",
+      keyword: "required",
+      params: { missingProperty: "attachments" },
+      message: "must have required property 'attachments'",
+    },
+  ]);
+});
+
+test("returns no errors if BASPI requirements are not met and null overlay is specified", () => {
+  const clonedVouch = JSON.parse(JSON.stringify(exampleVouch));
+  clonedVouch.claims = {
+    "/propertyPack/delayFactors": {
+      hasDelayFactors: { yesNo: "Yes" },
+    },
+  };
+  const errors = validateVerifiedClaims([clonedVouch], v3SchemaId, null);
+  expect(errors).toEqual([]);
+});
+
+test("returns errors for invalid fields even if null overlay is specified", () => {
+  const clonedVouch = JSON.parse(JSON.stringify(exampleVouch));
+  clonedVouch.claims = {
+    "/propertyPack/delayFactors": {
+      hasDelayFactors: { yesNo: "Maybe" },
+    },
+  };
+  const errors = validateVerifiedClaims([clonedVouch], v3SchemaId, null);
+  expect(errors).toHaveLength(4);
+});
+
+test("returns an array of errors for verified claim with multiple paths", () => {
+  const clonedVouch = JSON.parse(JSON.stringify(exampleVouch));
+  clonedVouch.claims = {
+    "/propertyPack/cousncilTaxBad": {
+      councilTaxBand: "D",
+      councilTaxAffectingAlterations: {
+        yesNo: "Yes",
+        details:
+          "Extension added in 2005 to add bedroom with ensuite shower room. Certificate of Compliance issued 17th Feb 2006 and council tax updated",
+      },
+    },
+    "/propertyPack/councilTaxBadTwo": {
+      councilTaxBand: "D",
+      councilTaxAffectingAlterations: {
+        yesNo: "Yes",
+        details:
+          "Extension added in 2005 to add bedroom with ensuite shower room. Certificate of Compliance issued 17th Feb 2006 and council tax updated",
+      },
+    },
+  };
+  expect(validateVerifiedClaims([clonedVouch], v3SchemaId, null)).toEqual([
+    "Path /propertyPack/cousncilTaxBad is not a valid PDTF schema path",
+    "Path /propertyPack/councilTaxBadTwo is not a valid PDTF schema path",
+  ]);
+});
+
+test("returns an empty array of errors for verified claim with multiple valid paths", () => {
+  const clonedVouch = JSON.parse(
+    JSON.stringify(exampleVouch),
+    v3SchemaId,
+    null
+  );
+  clonedVouch.claims = {
+    "/propertyPack/councilTax": {
+      councilTaxBand: "D",
+      councilTaxAffectingAlterations: {
+        yesNo: "Yes",
+        details:
+          "Extension added in 2005 to add bedroom with ensuite shower room. Certificate of Compliance issued 17th Feb 2006 and council tax updated",
+      },
+    },
+    "/propertyPack/address": {
+      line1: "property.line1",
+      line2: "property.line2",
+      town: "property.city",
+      county: "property.province",
+      postcode: "property.postcode",
+    },
+  };
+  expect(validateVerifiedClaims([clonedVouch], v3SchemaId, null)).toEqual([]);
+});
+
+test("returns an empty array for valid terms_of_use object with confidential level", () => {
+  const clonedVouch = JSON.parse(JSON.stringify(exampleVouch));
+  clonedVouch.terms_of_use = {
+    confidentiality_level: "confidential",
+    allowed_roles: ["Seller's Conveyancer", "Buyer's Conveyancer"]
+  };
+  expect(validateVerifiedClaims([clonedVouch])).toEqual([]);
+});
+
+test("returns an empty array for valid terms_of_use with public confidentiality", () => {
+  const clonedVouch = JSON.parse(JSON.stringify(exampleVouch));
+  clonedVouch.terms_of_use = {
+    confidentiality_level: "public",
+    allowed_roles: []
+  };
+  expect(validateVerifiedClaims([clonedVouch])).toEqual([]);
+});
+
+test("returns an empty array for valid terms_of_use with restricted level", () => {
+  const clonedVouch = JSON.parse(JSON.stringify(exampleVouch));
+  clonedVouch.terms_of_use = {
+    confidentiality_level: "restricted",
+    allowed_roles: []
+  };
+  expect(validateVerifiedClaims([clonedVouch])).toEqual([]);
+});
+
+test("returns errors for invalid confidentiality_level in terms_of_use", () => {
+  const clonedVouch = JSON.parse(JSON.stringify(exampleVouch));
+  clonedVouch.terms_of_use = {
+    confidentiality_level: "invalid_level",
+    allowed_roles: ["Seller's Conveyancer"]
+  };
+  const errors = validateVerifiedClaims([clonedVouch]);
+  expect(errors).toHaveLength(1);
+  expect(errors[0]).toHaveLength(1);
+  expect(errors[0][0].instancePath).toBe("/verified_claims/0/terms_of_use/confidentiality_level");
+  expect(errors[0][0].keyword).toBe("enum");
+});
+
+test("returns errors for invalid allowed_roles type in terms_of_use", () => {
+  const clonedVouch = JSON.parse(JSON.stringify(exampleVouch));
+  clonedVouch.terms_of_use = {
+    confidentiality_level: "confidential",
+    allowed_roles: "not_an_array"
+  };
+  const errors = validateVerifiedClaims([clonedVouch]);
+  expect(errors).toHaveLength(1);
+  expect(errors[0]).toHaveLength(1);
+  expect(errors[0][0].instancePath).toBe("/verified_claims/0/terms_of_use/allowed_roles");
+  expect(errors[0][0].keyword).toBe("type");
+});
+
+test("returns errors for additional properties in terms_of_use", () => {
+  const clonedVouch = JSON.parse(JSON.stringify(exampleVouch));
+  clonedVouch.terms_of_use = {
+    confidentiality_level: "confidential",
+    allowed_roles: ["Estate Agent"],
+    extra_property: "should_not_be_allowed"
+  };
+  const errors = validateVerifiedClaims([clonedVouch]);
+  expect(errors).toHaveLength(1);
+  expect(errors[0]).toHaveLength(1);
+  expect(errors[0][0].instancePath).toBe("/verified_claims/0/terms_of_use");
+  expect(errors[0][0].keyword).toBe("additionalProperties");
+});

package/charming-napier-57a77c/src/utils/compactSkeleton.js

@@ -0,0 +1,84 @@
+const fs = require("fs");
+
+// Read the current skeleton
+const skeleton = require("../schemas/v3/skeleton.json");
+
+// Convert to ultra-compact format
+const toCompact = (obj, indent = "") => {
+  if (typeof obj === "string") {
+    // Type indicators
+    if (["string", "number", "integer", "boolean", "variant"].includes(obj)) {
+      return obj[0]; // Just first letter: s, n, i, b, v
+    }
+    return obj;
+  }
+  
+  if (Array.isArray(obj)) {
+    if (obj.length === 0) return "[]";
+    // Array with content
+    const inner = toCompact(obj[0], indent + "  ");
+    if (typeof inner === "string" && inner.length === 1) {
+      return `[${inner}]`; // Simple array like [s]
+    }
+    return `[\n${indent}  ${inner}\n${indent}]`;
+  }
+  
+  if (obj && typeof obj === "object") {
+    const keys = Object.keys(obj);
+    if (keys.length === 0) return "{}";
+    
+    // Check if it's a simple object with only type indicators
+    const allSimple = keys.every(k => 
+      typeof obj[k] === "string" && 
+      ["s", "n", "i", "b", "v"].includes(toCompact(obj[k]))
+    );
+    
+    if (allSimple && keys.length < 5) {
+      // Inline format: {a:s b:n c:s}
+      return `{${keys.map(k => `${k}:${toCompact(obj[k])}`).join(" ")}}`;
+    }
+    
+    // Multi-line format
+    const lines = keys.map(k => {
+      const val = toCompact(obj[k], indent + "  ");
+      if (typeof val === "string" && val.length === 1) {
+        return `${indent}  ${k}:${val}`;
+      } else if (typeof val === "string" && (val === "{}" || val === "[]")) {
+        return `${indent}  ${k}${val}`;
+      } else if (typeof val === "string" && val.startsWith("{") && val.endsWith("}")) {
+        return `${indent}  ${k}${val}`;
+      } else if (typeof val === "string" && val.startsWith("[") && !val.includes("\n")) {
+        return `${indent}  ${k}${val}`;
+      } else {
+        return `${indent}  ${k}:\n${indent}  ${val}`;
+      }
+    });
+    
+    return `{\n${lines.join("\n")}\n${indent}}`;
+  }
+  
+  return String(obj);
+};
+
+// Generate ultra-compact version
+const compact = toCompact(skeleton);
+
+// Write compact version
+fs.writeFileSync("../schemas/v3/compactSkeleton.txt", compact);
+
+console.log("Compact skeleton written to ../schemas/v3/compactSkeleton.txt");
+console.log(`Size: ${compact.length} bytes`);
+
+// Count tokens
+try {
+  const GPT3Encoder = require('gpt-3-encoder');
+  const encoded = GPT3Encoder.encode(compact);
+  console.log(`Tokens (GPT-3): ${encoded.length}`);
+  console.log(`Reduction: ${((1 - encoded.length / 78734) * 100).toFixed(1)}% fewer tokens than JSON skeleton`);
+  
+  // Show sample
+  console.log("\nFirst 500 chars:");
+  console.log(compact.substring(0, 500));
+} catch (e) {
+  console.log(`Estimated tokens: ~${Math.ceil(compact.length / 4)}`);
+}

package/charming-napier-57a77c/src/utils/countTokens.js

@@ -0,0 +1,67 @@
+const fs = require('fs');
+
+// Try to use tiktoken or a similar library
+// For Claude/Anthropic models, we can estimate using the cl100k_base encoding
+// which is similar to what Claude uses
+
+// First, let's check what's available
+try {
+  // Option 1: Use gpt-tokenizer (common npm package)
+  const { encode } = require('gpt-tokenizer');
+  
+  const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+  const tokens = encode(content);
+  
+  console.log(`Using gpt-tokenizer:`);
+  console.log(`Total tokens: ${tokens.length}`);
+  console.log(`File size: ${content.length} characters`);
+  console.log(`Ratio: ${(content.length / tokens.length).toFixed(2)} characters per token`);
+  
+} catch (e1) {
+  console.log("gpt-tokenizer not found, trying gpt-3-encoder...");
+  
+  try {
+    // Option 2: Use gpt-3-encoder
+    const GPT3Encoder = require('gpt-3-encoder');
+    
+    const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+    const encoded = GPT3Encoder.encode(content);
+    
+    console.log(`Using gpt-3-encoder:`);
+    console.log(`Total tokens: ${encoded.length}`);
+    console.log(`File size: ${content.length} characters`);
+    console.log(`Ratio: ${(content.length / encoded.length).toFixed(2)} characters per token`);
+    
+  } catch (e2) {
+    console.log("No tokenizer library found. Falling back to estimation...");
+    
+    // Option 3: Manual estimation
+    const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+    
+    // Count different types of content for better estimation
+    const lines = content.split('\n');
+    const words = content.split(/\s+/);
+    const punctuation = content.match(/[{}:,\[\]"]/g) || [];
+    
+    // Rough estimation:
+    // - JSON structure tokens (brackets, colons, quotes): ~1 token each
+    // - Words: ~1 token each
+    // - Whitespace: usually absorbed into adjacent tokens
+    
+    const structureTokens = punctuation.length;
+    const wordTokens = words.length;
+    const estimatedTokens = Math.ceil((structureTokens + wordTokens) * 0.75); // 0.75 factor for overlap
+    
+    console.log("Manual estimation:");
+    console.log(`File size: ${content.length} characters`);
+    console.log(`Lines: ${lines.length}`);
+    console.log(`Words: ${words.length}`);
+    console.log(`JSON punctuation: ${punctuation.length}`);
+    console.log(`Estimated tokens: ~${estimatedTokens}`);
+    console.log(`Ratio: ${(content.length / estimatedTokens).toFixed(2)} characters per token`);
+    
+    // Also do a simple character-based estimation
+    const simpleEstimate = Math.ceil(content.length / 4);
+    console.log(`\nSimple estimation (chars/4): ~${simpleEstimate} tokens`);
+  }
+}