@pdtf/schemas 3.6.0-dev.3 → 3.6.0-dev.5

package/compassionate-easley-bafb14/src/schemas/verifiedClaims/README.md

@@ -0,0 +1,21 @@
+## README
+
+### About verified claims
+`pdtf-verified-claims.json` describes the schema container in which we store property `claims` and their corresponding `verification`.
+
+This schema structure is strongly influenced by OpenID Connect for Identity Assurance 1.0's way of storing "verified claims".
+
+More Details from the OIDC page which describes the details the nature of `verification` and `claims` objects: https://openid.net/specs/openid-connect-4-identity-assurance-1_0.html#name-representing-verified-claim
+
+### Terms of Use
+The schema supports an optional `terms_of_use` object to control access to verified claims based on confidentiality levels:
+
+* **Public**: No access restrictions (`allowed_roles: []`)
+* **Restricted**: Requires some participant role but not specifically defined (`allowed_roles: []`)
+* **Confidential**: Specific roles defined for access control (e.g., `allowed_roles: ["Seller's Conveyancer", "Buyer's Conveyancer"]`)
+
+The `allowed_roles` array accepts any of the participant roles defined in [pdtf-transaction.json](../v3/pdtf-transaction.json).
+
+Notes:
+* For PDTF Schema, all data included in the `claims` object must adhere to [pdtf-transaction.json (v3)](../v3/pdtf-transaction.json)
+* For verification, there is more freedom in terms of the type of data that can currently be provided, see the documentation in the OIDC link above for details and examples.

package/compassionate-easley-bafb14/src/schemas/verifiedClaims/pdtf-verified-claims.json

@@ -0,0 +1,447 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://trust.propdata.org.uk/schemas/verifiedClaims/pdtf-verified-claims.json",
+  "definitions": {
+    "date_type": {
+      "type": "string",
+      "format": "date"
+    },
+    "time_type": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "document_number": {
+      "type": "string"
+    },
+    "document_details": {
+      "type": "object",
+      "properties": {
+        "type": {
+          "type": "string"
+        },
+        "document_number": {
+          "$ref": "#/definitions/document_number"
+        },
+        "number": {
+          "$ref": "#/definitions/document_number"
+        },
+        "personal_number": {
+          "type": "string"
+        },
+        "serial_number": {
+          "type": "string"
+        },
+        "date_of_issuance": {
+          "$ref": "#/definitions/date_type"
+        },
+        "date_of_expiry": {
+          "$ref": "#/definitions/date_type"
+        },
+        "issuer": {
+          "type": "object",
+          "properties": {
+            "name": {
+              "type": "string"
+            },
+            "formatted": {
+              "type": "string"
+            },
+            "street_address": {
+              "type": "string"
+            },
+            "locality": {
+              "type": "string"
+            },
+            "region": {
+              "type": "string"
+            },
+            "postal_code": {
+              "type": "string"
+            },
+            "country": {
+              "type": "string"
+            },
+            "country_code": {
+              "type": "string"
+            },
+            "jurisdiction": {
+              "type": "string"
+            }
+          }
+        }
+      }
+    },
+    "validation_method": {
+      "type": "object",
+      "properties": {
+        "type": {
+          "type": "string"
+        }
+      }
+    },
+    "verification_method": {
+      "type": "object",
+      "properties": {
+        "type": {
+          "type": "string"
+        }
+      }
+    },
+    "evidence": {
+      "type": "object",
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": ["document", "electronic_record", "vouch"]
+        },
+        "attachments": {
+          "$ref": "#/definitions/attachments"
+        }
+      },
+      "required": ["type"],
+      "allOf": [
+        {
+          "if": {
+            "properties": {
+              "type": {
+                "const": "document"
+              }
+            }
+          },
+          "then": {
+            "properties": {
+              "validation_method": {
+                "$ref": "#/definitions/validation_method"
+              },
+              "verification_method": {
+                "$ref": "#/definitions/verification_method"
+              },
+              "verifier": {
+                "type": "object",
+                "properties": {
+                  "organization": {
+                    "type": "string"
+                  },
+                  "txn": {
+                    "type": "string"
+                  }
+                }
+              },
+              "time": {
+                "$ref": "#/definitions/time_type"
+              },
+              "document_details": {
+                "$ref": "#/definitions/document_details"
+              }
+            }
+          }
+        },
+        {
+          "if": {
+            "properties": {
+              "type": {
+                "value": "electronic_record"
+              }
+            }
+          },
+          "then": {
+            "properties": {
+              "validation_method": {
+                "$ref": "#/definitions/validation_method"
+              },
+              "verification_method": {
+                "$ref": "#/definitions/verification_method"
+              },
+              "verifier": {
+                "type": "object",
+                "required": ["organization"],
+                "properties": {
+                  "organization": {
+                    "type": "string"
+                  },
+                  "txn": {
+                    "type": "string"
+                  }
+                }
+              },
+              "time": {
+                "$ref": "#/definitions/time_type"
+              },
+              "record": {
+                "type": "object",
+                "required": ["source"],
+                "properties": {
+                  "type": {
+                    "type": "string"
+                  },
+                  "personal_number": {
+                    "type": "string"
+                  },
+                  "created_at": {
+                    "$ref": "#/definitions/date_type"
+                  },
+                  "date_of_expiry": {
+                    "$ref": "#/definitions/date_type"
+                  },
+                  "source": {
+                    "type": "object",
+                    "required": ["name"],
+                    "properties": {
+                      "name": {
+                        "type": "string"
+                      },
+                      "formatted": {
+                        "type": "string"
+                      },
+                      "street_address": {
+                        "type": "string"
+                      },
+                      "locality": {
+                        "type": "string"
+                      },
+                      "region": {
+                        "type": "string"
+                      },
+                      "postal_code": {
+                        "type": "string"
+                      },
+                      "country": {
+                        "type": "string"
+                      },
+                      "country_code": {
+                        "type": "string"
+                      },
+                      "jurisdiction": {
+                        "type": "string"
+                      },
+                      "rights": { "type": "string" }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        {
+          "if": {
+            "properties": {
+              "type": {
+                "value": "vouch"
+              }
+            }
+          },
+          "then": {
+            "properties": {
+              "validation_method": {
+                "$ref": "#/definitions/validation_method"
+              },
+              "verification_method": {
+                "$ref": "#/definitions/verification_method"
+              },
+              "verifier": {
+                "type": "object",
+                "properties": {
+                  "organization": {
+                    "type": "string"
+                  },
+                  "txn": {
+                    "type": "string"
+                  }
+                }
+              },
+              "time": {
+                "$ref": "#/definitions/time_type"
+              },
+              "attestation": {
+                "type": "object",
+                "properties": {
+                  "type": {
+                    "type": "string",
+                    "const": "digital_attestation"
+                  },
+                  "reference_number": {
+                    "type": "string"
+                  },
+                  "personal_number": {
+                    "type": "string"
+                  },
+                  "date_of_issuance": {
+                    "$ref": "#/definitions/date_type"
+                  },
+                  "date_of_expiry": {
+                    "$ref": "#/definitions/date_type"
+                  },
+                  "voucher": {
+                    "type": "object",
+                    "properties": {
+                      "name": {
+                        "type": "string"
+                      },
+                      "birthdate": {
+                        "$ref": "#/definitions/date_type"
+                      },
+                      "formatted": {
+                        "type": "string"
+                      },
+                      "street_address": {
+                        "type": "string"
+                      },
+                      "locality": {
+                        "type": "string"
+                      },
+                      "region": {
+                        "type": "string"
+                      },
+                      "postal_code": {
+                        "type": "string"
+                      },
+                      "country": {
+                        "type": "string"
+                      },
+                      "occupation": {
+                        "type": "string"
+                      },
+                      "organization": {
+                        "type": "string"
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      ]
+    },
+    "digest": {
+      "type": "object",
+      "properties": {
+        "alg": {
+          "type": "string"
+        },
+        "value": {
+          "type": "string",
+          "contentEncoding": "base64"
+        }
+      },
+      "required": ["alg", "value"]
+    },
+    "external_attachment": {
+      "type": "object",
+      "properties": {
+        "desc": {
+          "type": "string"
+        },
+        "digest": {
+          "$ref": "#/definitions/digest"
+        },
+        "url": {
+          "type": "string",
+          "format": "uri"
+        },
+        "access_token": {
+          "type": ["string", "null"]
+        },
+        "expires_in": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "pdtfSchemaPath": {
+          "type": "string"
+        }
+      },
+      "required": ["digest", "url"]
+    },
+    "attachments": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "$ref": "#/definitions/external_attachment"
+      }
+    },
+    "verified_claims_def": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "description": "The unique identifier of the verified claim",
+          "type": "string"
+        },
+        "transactionId": {
+          "description": "The unique identifier of the PDTF transaction to which these claims relate",
+          "type": "string"
+        },
+        "schemaVersion": {
+          "description": "The semantic version of the schema used to validate the claims",
+          "type": "string"
+        },
+        "timestamp": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "Unix timestamp in milliseconds when the verified claims were created"
+        },
+        "verification": {
+          "type": "object",
+          "properties": {
+            "trust_framework": {
+              "type": "string",
+              "const": "uk_pdtf"
+            },
+            "time": {
+              "$ref": "#/definitions/time_type"
+            },
+            "evidence": {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "oneOf": [
+                  {
+                    "$ref": "#/definitions/evidence"
+                  }
+                ]
+              }
+            }
+          },
+          "required": ["trust_framework"],
+          "additionalProperties": true
+        },
+        "claims": {
+          "type": "object",
+          "patternProperties": {
+            "^(/|([a-zA-Z0-9/-]+)+$)": {}
+          },
+          "minProperties": 1,
+          "additionalProperties": false
+        },
+        "terms_of_use": {
+          "type": "object",
+          "properties": {
+            "confidentiality_level": {
+              "type": "string",
+              "enum": ["public", "restricted", "confidential"]
+            },
+            "allowed_roles": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          },
+          "additionalProperties": false
+        }
+      },
+      "required": ["verification", "claims"],
+      "additionalProperties": false
+    }
+  },
+  "type": "object",
+  "required": ["verified_claims"],
+  "properties": {
+    "verified_claims": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/verified_claims_def"
+      }
+    }
+  },
+  "additionalProperties": false
+}

package/compassionate-easley-bafb14/src/utils/compactSkeleton.js

@@ -0,0 +1,84 @@
+const fs = require("fs");
+
+// Read the current skeleton
+const skeleton = require("../schemas/v3/skeleton.json");
+
+// Convert to ultra-compact format
+const toCompact = (obj, indent = "") => {
+  if (typeof obj === "string") {
+    // Type indicators
+    if (["string", "number", "integer", "boolean", "variant"].includes(obj)) {
+      return obj[0]; // Just first letter: s, n, i, b, v
+    }
+    return obj;
+  }
+  
+  if (Array.isArray(obj)) {
+    if (obj.length === 0) return "[]";
+    // Array with content
+    const inner = toCompact(obj[0], indent + "  ");
+    if (typeof inner === "string" && inner.length === 1) {
+      return `[${inner}]`; // Simple array like [s]
+    }
+    return `[\n${indent}  ${inner}\n${indent}]`;
+  }
+  
+  if (obj && typeof obj === "object") {
+    const keys = Object.keys(obj);
+    if (keys.length === 0) return "{}";
+    
+    // Check if it's a simple object with only type indicators
+    const allSimple = keys.every(k => 
+      typeof obj[k] === "string" && 
+      ["s", "n", "i", "b", "v"].includes(toCompact(obj[k]))
+    );
+    
+    if (allSimple && keys.length < 5) {
+      // Inline format: {a:s b:n c:s}
+      return `{${keys.map(k => `${k}:${toCompact(obj[k])}`).join(" ")}}`;
+    }
+    
+    // Multi-line format
+    const lines = keys.map(k => {
+      const val = toCompact(obj[k], indent + "  ");
+      if (typeof val === "string" && val.length === 1) {
+        return `${indent}  ${k}:${val}`;
+      } else if (typeof val === "string" && (val === "{}" || val === "[]")) {
+        return `${indent}  ${k}${val}`;
+      } else if (typeof val === "string" && val.startsWith("{") && val.endsWith("}")) {
+        return `${indent}  ${k}${val}`;
+      } else if (typeof val === "string" && val.startsWith("[") && !val.includes("\n")) {
+        return `${indent}  ${k}${val}`;
+      } else {
+        return `${indent}  ${k}:\n${indent}  ${val}`;
+      }
+    });
+    
+    return `{\n${lines.join("\n")}\n${indent}}`;
+  }
+  
+  return String(obj);
+};
+
+// Generate ultra-compact version
+const compact = toCompact(skeleton);
+
+// Write compact version
+fs.writeFileSync("../schemas/v3/compactSkeleton.txt", compact);
+
+console.log("Compact skeleton written to ../schemas/v3/compactSkeleton.txt");
+console.log(`Size: ${compact.length} bytes`);
+
+// Count tokens
+try {
+  const GPT3Encoder = require('gpt-3-encoder');
+  const encoded = GPT3Encoder.encode(compact);
+  console.log(`Tokens (GPT-3): ${encoded.length}`);
+  console.log(`Reduction: ${((1 - encoded.length / 78734) * 100).toFixed(1)}% fewer tokens than JSON skeleton`);
+  
+  // Show sample
+  console.log("\nFirst 500 chars:");
+  console.log(compact.substring(0, 500));
+} catch (e) {
+  console.log(`Estimated tokens: ~${Math.ceil(compact.length / 4)}`);
+}

package/compassionate-easley-bafb14/src/utils/countTokens.js

@@ -0,0 +1,67 @@
+const fs = require('fs');
+
+// Try to use tiktoken or a similar library
+// For Claude/Anthropic models, we can estimate using the cl100k_base encoding
+// which is similar to what Claude uses
+
+// First, let's check what's available
+try {
+  // Option 1: Use gpt-tokenizer (common npm package)
+  const { encode } = require('gpt-tokenizer');
+  
+  const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+  const tokens = encode(content);
+  
+  console.log(`Using gpt-tokenizer:`);
+  console.log(`Total tokens: ${tokens.length}`);
+  console.log(`File size: ${content.length} characters`);
+  console.log(`Ratio: ${(content.length / tokens.length).toFixed(2)} characters per token`);
+  
+} catch (e1) {
+  console.log("gpt-tokenizer not found, trying gpt-3-encoder...");
+  
+  try {
+    // Option 2: Use gpt-3-encoder
+    const GPT3Encoder = require('gpt-3-encoder');
+    
+    const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+    const encoded = GPT3Encoder.encode(content);
+    
+    console.log(`Using gpt-3-encoder:`);
+    console.log(`Total tokens: ${encoded.length}`);
+    console.log(`File size: ${content.length} characters`);
+    console.log(`Ratio: ${(content.length / encoded.length).toFixed(2)} characters per token`);
+    
+  } catch (e2) {
+    console.log("No tokenizer library found. Falling back to estimation...");
+    
+    // Option 3: Manual estimation
+    const content = fs.readFileSync('../schemas/v3/skeleton.json', 'utf8');
+    
+    // Count different types of content for better estimation
+    const lines = content.split('\n');
+    const words = content.split(/\s+/);
+    const punctuation = content.match(/[{}:,\[\]"]/g) || [];
+    
+    // Rough estimation:
+    // - JSON structure tokens (brackets, colons, quotes): ~1 token each
+    // - Words: ~1 token each
+    // - Whitespace: usually absorbed into adjacent tokens
+    
+    const structureTokens = punctuation.length;
+    const wordTokens = words.length;
+    const estimatedTokens = Math.ceil((structureTokens + wordTokens) * 0.75); // 0.75 factor for overlap
+    
+    console.log("Manual estimation:");
+    console.log(`File size: ${content.length} characters`);
+    console.log(`Lines: ${lines.length}`);
+    console.log(`Words: ${words.length}`);
+    console.log(`JSON punctuation: ${punctuation.length}`);
+    console.log(`Estimated tokens: ~${estimatedTokens}`);
+    console.log(`Ratio: ${(content.length / estimatedTokens).toFixed(2)} characters per token`);
+    
+    // Also do a simple character-based estimation
+    const simpleEstimate = Math.ceil(content.length / 4);
+    console.log(`\nSimple estimation (chars/4): ~${simpleEstimate} tokens`);
+  }
+}