@pdtf/schemas 3.6.0-dev.7 → 3.6.0-dev.8

package/busy-spence-79c6bf/src/utils/pathSkeleton.js

@@ -0,0 +1,181 @@
+const fs = require("fs");
+const traverse = require("traverse");
+
+const combinedSchema = require("../schemas/v3/combined.json");
+
+// Extract fields that have overlay-specific properties
+const extractFields = [
+  "baspi4", "baspi5", "nts", "nts2", "ntsl", "ntsl2",
+  "ta6", "ta7", "ta7ed5", "ta10", "lpe1", "fme1", "piq",
+  "con29R", "con29DW", "llc1", "rds", "oc1", "sr24",
+];
+
+// Delete properties we don't want in the skeleton
+const deleteProperties = (sourceSchema, propertyNames) => {
+  traverse(sourceSchema).forEach(function (element) {
+    if (propertyNames.includes(this.key)) {
+      this.delete(true);
+    }
+  });
+  return sourceSchema;
+};
+
+// Collect all paths in path notation
+const collectPaths = (schema, currentPath = "") => {
+  const paths = [];
+  
+  if (!schema || typeof schema !== 'object') {
+    return paths;
+  }
+  
+  // Handle arrays
+  if (schema.type === "array" && schema.items) {
+    const arrayPath = currentPath + "[]";
+    paths.push(arrayPath);
+    // Continue with items
+    const itemPaths = collectPaths(schema.items, arrayPath);
+    paths.push(...itemPaths);
+    return paths;
+  }
+  
+  // Handle primitives - add type suffix
+  if (schema.type && !schema.properties && !schema.oneOf && !schema.items) {
+    let typeChar = "";
+    switch(schema.type) {
+      case "string": typeChar = ":s"; break;
+      case "number": typeChar = ":n"; break;
+      case "integer": typeChar = ":i"; break;
+      case "boolean": typeChar = ":b"; break;
+      default: typeChar = "";
+    }
+    if (currentPath) {
+      paths.push(currentPath + typeChar);
+    }
+    return paths;
+  }
+  
+  // Handle objects with properties
+  if (schema.properties) {
+    // If this is an object without a path yet, don't add it
+    if (currentPath && !schema.type?.includes("array")) {
+      paths.push(currentPath);
+    }
+    
+    Object.keys(schema.properties).forEach((propName) => {
+      const newPath = currentPath ? `${currentPath}.${propName}` : propName;
+      const propPaths = collectPaths(schema.properties[propName], newPath);
+      paths.push(...propPaths);
+    });
+  }
+  
+  // Handle oneOf - collect all possible properties
+  if (schema.oneOf) {
+    const variantProps = new Set();
+    schema.oneOf.forEach((oneOfSchema) => {
+      if (oneOfSchema.properties) {
+        Object.keys(oneOfSchema.properties).forEach((propName) => {
+          variantProps.add(propName);
+        });
+      }
+    });
+    
+    // Process each unique property
+    variantProps.forEach((propName) => {
+      const newPath = currentPath ? `${currentPath}.${propName}` : propName;
+      // Check if all variants have the same type
+      let commonType = null;
+      let isVariant = false;
+      
+      schema.oneOf.forEach((oneOfSchema) => {
+        if (oneOfSchema.properties && oneOfSchema.properties[propName]) {
+          const propType = oneOfSchema.properties[propName].type;
+          if (commonType === null) {
+            commonType = propType;
+          } else if (commonType !== propType) {
+            isVariant = true;
+          }
+        }
+      });
+      
+      if (isVariant || !commonType) {
+        paths.push(newPath + ":v"); // variant
+      } else {
+        // Process the first variant's version of this property
+        for (const oneOfSchema of schema.oneOf) {
+          if (oneOfSchema.properties && oneOfSchema.properties[propName]) {
+            const propPaths = collectPaths(oneOfSchema.properties[propName], newPath);
+            paths.push(...propPaths);
+            break;
+          }
+        }
+      }
+    });
+  }
+  
+  // Empty objects
+  if (currentPath && !schema.properties && !schema.oneOf && !schema.items && !schema.type) {
+    paths.push(currentPath);
+  }
+  
+  return paths;
+};
+
+// Remove duplicates
+const deduplicatePaths = (paths) => {
+  return [...new Set(paths)];
+};
+
+// Sort paths for readability
+const sortPaths = (paths) => {
+  return paths.sort((a, b) => {
+    // Sort by depth first (fewer dots = higher up)
+    const depthA = (a.match(/\./g) || []).length;
+    const depthB = (b.match(/\./g) || []).length;
+    if (depthA !== depthB) return depthA - depthB;
+    
+    // Then alphabetically
+    return a.localeCompare(b);
+  });
+};
+
+// Create clean schema
+const coreSchema = deleteProperties(JSON.parse(JSON.stringify(combinedSchema)), [
+  "discriminator",
+  ...extractFields.map((item) => `${item}Ref`),
+  ...extractFields.map((item) => `${item}Required`),
+  ...extractFields.map((item) => `${item}Title`),
+  ...extractFields.map((item) => `${item}Description`),
+  ...extractFields.map((item) => `${item}Enum`),
+]);
+
+// Remove metadata
+const skeletonSchema = deleteProperties(coreSchema, [
+  "$schema", "$id", "title", "description", "required",
+  "enum", "minItems", "minLength", "format", "minimum", "maximum",
+]);
+
+// Collect paths
+const allPaths = collectPaths(skeletonSchema);
+const uniquePaths = deduplicatePaths(allPaths);
+const sortedPaths = sortPaths(uniquePaths);
+
+// Write as text file
+const pathContent = sortedPaths.join('\n');
+fs.writeFileSync("../schemas/v3/pathSkeleton.txt", pathContent);
+
+// Also create a JSON version for comparison
+fs.writeFileSync("../schemas/v3/pathSkeleton.json", JSON.stringify(sortedPaths, null, 2));
+
+console.log(`Path skeleton written to ../schemas/v3/pathSkeleton.txt`);
+console.log(`Total paths: ${sortedPaths.length}`);
+console.log(`Text size: ${pathContent.length} bytes`);
+
+// Count tokens
+try {
+  const GPT3Encoder = require('gpt-3-encoder');
+  const encoded = GPT3Encoder.encode(pathContent);
+  console.log(`Tokens (GPT-3): ${encoded.length}`);
+  console.log(`Reduction: ${((1 - encoded.length / 78734) * 100).toFixed(1)}% fewer tokens than JSON skeleton`);
+} catch (e) {
+  console.log(`Estimated tokens: ~${Math.ceil(pathContent.length / 4)}`);
+}

package/busy-spence-79c6bf/ta6ed6-titles-plan.md

@@ -0,0 +1,118 @@
+# Plan: Exact TA6 6th-edition field titles (`ta6ed6Title`) in combined.json
+
+## Context
+
+The Law Society requires that when a TA6 form is rendered digitally, field labels match the
+published form **exactly**. The PDTF schema already carries form-specific labels for other forms
+via `<form>Title` properties (e.g. `ta6Title`, `ntsTitle`). The TA6 6th-edition (2025) layer was
+recently added: `combined.json` now has **537 `ta6ed6Ref`** markers identifying which schema
+properties belong to the 6th-edition form, and **51 `ta6ed6Title`** values already applied. The
+remaining 6th-edition fields still fall back to the generic base `title`, whose wording differs
+from the Law Society form.
+
+This change completes the `ta6ed6Title` coverage so that **every** 6th-edition field whose form
+wording differs from the base `title` carries its exact Law Society label, verified against the
+source form PDF.
+
+## Architecture (verified)
+
+- **Source of truth:** `src/schemas/v3/combined.json` is hand-edited. It is the only place the 51
+  existing `ta6ed6Title` values live (the `ta6ed6.json` overlay has the 537 refs but 0 titles —
+  it is simply stale).
+- **Generated artifacts:** `src/utils/extractOverlay.js` reads `combined.json` and regenerates the
+  per-form overlay files (incl. `overlays/ta6ed6.json`) and the core schema
+  (`pdtf-transaction.json`). Its `extractFields` list already includes `"ta6ed6"`, so it extracts
+  `ta6ed6Ref`, `ta6ed6Title`, `ta6ed6Required`, `ta6ed6Enum`, `ta6ed6Type`, etc. Running it after
+  editing propagates the new titles into the overlay.
+- **Field identity:** a property belongs to the 6th-edition form iff it has a `ta6ed6Ref`. The ref
+  value (e.g. `10.2`, `5.9a`) matches the `ref` keys in the extracted form file
+  `tmp/ta6ed6-fields.json`, which was validated against the PDF (e.g. ref `10.2` →
+  "Is a permit required for on-road parking?").
+
+## Authoritative title sources
+
+1. `tmp/TA6 Form 6th edition.pdf` — the Law Society source form (20 pages, text-based). **The
+   final authority** per the user's "re-verify all against PDF" decision.
+2. `tmp/ta6ed6-fields.json` — clean extraction keyed by `ref` (16 sections, ~195 labelled fields).
+   Used as the working map; every title applied must be reconciled against the PDF text.
+
+## House style for titles (from `tmp/ta6ed6-mapping-report.md` §C.5 — matches the existing 51)
+
+A `ta6ed6Title` is the exact form question/field text, with:
+1. Sub-question prefixes stripped — `"(a) "`, `"(b) "`, `"(i) "`, `"(ii) "` (hierarchy is carried
+   by `ta6ed6Ref` such as `5.6a`, `5.6g(i)`).
+2. Conditional prefixes stripped — `"If yes, "`, `"If no, "` (implicit from the `oneOf` branch).
+3. First letter capitalised after any stripping.
+Otherwise verbatim — punctuation, casing, and wording must match the form character-for-character.
+
+## Scope decision (confirmed with user)
+
+- **Base branch:** rebase this work onto `dev` (which holds the 537 refs + 51 titles + the
+  `ta6ed6.json` overlay). This worktree is currently off `master`, which has none of the ta6ed6
+  layer. PR target: `dev`.
+- **Which fields:** add `ta6ed6Title` **only where the 6th-edition label differs** from the
+  property's existing base `title`. Skip fields whose base `title` already matches the form.
+  Also re-verify the existing 51 against the PDF and correct any that are wrong.
+- **Sub-components** (`yesNo`, `.1`/`.2` answer parts that have no distinct printed label) inherit
+  the parent question label and do **not** receive their own `ta6ed6Title`.
+
+## Steps
+
+### 1. Rebase onto dev
+The worktree branch `claude/busy-spence-79c6bf` equals `master` (clean, no work commits), so:
+`git reset --hard dev`. Confirm `combined.json` then shows 537 `ta6ed6Ref` / 51 `ta6ed6Title`.
+
+### 2. Build the canonical title map
+- Read the full PDF (pages 1–20) and transcribe the exact label for every form ref.
+- Load `tmp/ta6ed6-fields.json` and reconcile each entry against the transcribed PDF text;
+  correct the JSON map where the extraction and PDF disagree (PDF wins).
+- Produce an in-memory map `ta6ed6Ref → canonicalTitle` (after applying the §C.5 house style).
+
+### 3. Apply titles to `combined.json`
+Walk `combined.json`. For each object carrying a `ta6ed6Ref` that maps to a canonical label:
+- If `canonicalTitle !== base title` → set/replace `ta6ed6Title` with the canonical value, placed
+  immediately **after** the `title` property (matching the existing convention:
+  `…refs, "title", "ta6ed6Title", "type"…`).
+- If `canonicalTitle === base title` → ensure no redundant `ta6ed6Title` is present.
+- Re-verify each of the existing 51 and correct mismatches.
+Preserve JSON formatting (2-space indent, key order) to keep the diff reviewable. Where a single
+form field maps to two refs (e.g. `documents` + `attachments` sharing a ref, per §C.4), title the
+user-facing question property, not the attachment helper.
+
+### 4. Regenerate derived artifacts
+Run `node src/utils/extractOverlay.js` so the new titles propagate into `overlays/ta6ed6.json`
+(and refresh the core schema). Confirm `overlays/ta6ed6.json` `ta6ed6Title` count now matches the
+number of titled fields. Regenerate skeletons too if the repo's release flow does (see recent
+commit "Regenerate overlays and skeletons").
+
+## Verification
+
+1. **Exactness audit script (read-only):** generate a table of every `ta6ed6Ref` → applied
+   `ta6ed6Title` (or inherited base `title`) and diff it against the PDF-derived canonical map.
+   Output must show **zero** unexplained mismatches. Save the report alongside the existing
+   `tmp/ta6ed6-mapping-report.md` for reviewer sign-off.
+2. **Schema validity:** `npm test` (jest) — all suites green, especially
+   `src/tests/v3/ta6ed6.test.js` and `transactionSchema.test.js`.
+3. **Overlay parity:** assert every `ta6ed6Title` in `combined.json` is present in the regenerated
+   `overlays/ta6ed6.json` (proves the generation step captured them).
+4. **Spot render check:** optionally load a ta6ed6 transaction via `getTransactionSchema` /
+   `getValidator` to confirm titles surface as expected.
+
+## Critical files
+
+- `src/schemas/v3/combined.json` — primary edit target.
+- `src/schemas/v3/overlays/ta6ed6.json`, `src/schemas/v3/pdtf-transaction.json` — regenerated, not
+  hand-edited.
+- `src/utils/extractOverlay.js` — generation script (already lists `ta6ed6`).
+- `tmp/TA6 Form 6th edition.pdf`, `tmp/ta6ed6-fields.json`, `tmp/ta6ed6-mapping-report.md` —
+  title sources / reference (git-ignored; not committed).
+- `src/tests/v3/ta6ed6.test.js` — existing test coverage.
+
+## Open considerations
+
+- The audit in step-4/verification may surface base-schema labels that are *already* exactly right
+  for the form (no `ta6ed6Title` needed) — these are correctly left untouched under the
+  "only where wording differs" decision.
+- A handful of refs are overlay-only discriminators / aggregations with no printed form label
+  (e.g. `areBoundariesUniform`, `isInsured`, `offMainsDrainageSystemType`; see mapping report
+  §C.1). These get no `ta6ed6Title`.

package/busy-spence-79c6bf/ta7ed5-titles-spec.md

@@ -0,0 +1,89 @@
+# Spec: Exact TA7 5th-edition field titles (`ta7ed5Title`)
+
+## Context
+
+Mirror of the TA6 6th-edition work: add `ta7ed5Title` properties carrying the verbatim
+Law Society *TA7 Leasehold Information Form (5th edition) (2025)* labels, so the form renders
+with compliant labelling. Source verified against `TA7-5th-edition.pdf` (9 pages, 12 sections),
+extracted with `pdftotext`.
+
+## Audit (current state on `dev`)
+
+- `ta7ed5Ref` markers in `combined.json`: **156**
+- Existing `ta7ed5Title`: **20** (sections 6, 7, 10, 11 mostly done; some retain trailing colons)
+- Candidates (non-empty base title, no `ta7ed5Title`): **109**
+- Empty base title (yesNo/answer components — inherit, no title): 27
+- Overlay `overlays/ta7ed5.json` exists; titles propagate via `extractOverlay.js` as `title`.
+
+## House style (same as TA6, confirmed)
+
+Strip `(a)`/`(i)` and `If yes,`/`If no,` prefixes; omit trailing colons; **no override when a
+colon is the only difference**; `ta7ed5Title` placed immediately after `title`. Use the form's
+exact wording incl. its curly apostrophes (e.g. landlord’s). Also re-verify the existing 20 and
+strip their trailing colons (4 have them: 8.1.2, 11.2.2, 11.4.2, 11.6.2).
+
+## Proposed changes (clear adds — form question/field wording differs from base)
+
+| Ref | Property | ta7ed5Title (exact form text, normalised) |
+|-----|----------|-------------------------------------------|
+| 1.1 | address | Address of the property |
+| 1.1.3 | uprn | UPRN (if known) |
+| 4.4.1 | sellerOwnership.yesNo | Does the seller alone, or jointly with others, own the freehold, any headlease or a management company named in the lease? |
+| 4.4.2 | sellerOwnership.attachments | Supply evidence of your ownership or share certificate for membership of the organisation that is responsible for the management of the building. |
+| 5.3 | sellerContributesToServiceCharge | Do you contribute to the cost of maintaining the building? |
+| 5.4 | largeAdditionalExpense | Do you know of any expense (e.g. the cost of redecoration of outside or communal areas that is not usually incurred annually) that is likely to be shown in the service charge accounts within the next three years? |
+| 5.4.2 | details | Please give details |
+| 5.5 | serviceChargeChallenged | Have you challenged the service charge or any expense in the last three years? |
+| 5.6 | problemInServiceChargeLevel | Do you know of any issues in the last three years regarding the level of service charges or with the management? |
+| 7.3 | sellerAwareOfResponse | Are you aware of any response to a notice disclosed in replies to 7.1 and 7.2 above? |
+| 8.1 | changesInTermOfLease | Are you aware of any changes to the terms of the lease? |
+| 9.1 | sellerAwareOfAlterations | Are you aware of any alterations having been made to the property since the lease was originally granted? |
+
+(8.2 `landlordConsents`, 8.2.2 `details`, 9.2 `details`, 9.3 `landlordConsentObtained` already match the
+form once colon/case-normalised → no override; confirmed at implementation.)
+
+## Decisions to confirm (judgment calls — see questions)
+
+**A. Section / sub-group headings** where the schema container name differs from the form heading:
+- 3 `contactDetails` "Contact Details" → **"Contact details"**
+- 3.1.2 `managementCompany` "Management Company" → **"Management company"**
+- 10 `disputes` "Disputes" → **"Complaints"** (form section 10 is "Complaints")
+- 11 `buildingSafetyAct` "Building safety" → **"Building safety for blocks of flats"**
+
+**B. Per-contact field labels** (forms label these "Name:" / "Email:"), 5 contacts in §3 + managing
+agent in §4 — ~14 tiny fields:
+- `nameOrOrganisation` "Name or organisation" → **"Name"**
+- `emailAddress` "Email address" → **"Email"**
+- (`address`/`postcode`/`line1` already match or have no distinct form label → skip)
+
+**C. "Who collects" questions (§4.2/4.3)** — schema frames these as *who collects*, the form asks a
+yes/no first:
+- 4.2 `collectsbuildingInsurancePremiums` → form 4.2 "Does the managing agent arrange or collect charges for the insurance of the building?"
+- 4.3 `collectsGroundRent` → form 4.3 "If ground rent is payable, is it collected by the managing agent?"
+- (sub-collectors `collector`/`otherCollector` → leave, or map to form's "Other (please provide details)")
+
+**D. §2.2 payment-demand fields** — form 2.2 is one question with (a)/(b)/(c) sub-items:
+- Option 1: label the demand-status fields with sub-item names — 2.2a → "Ground rent", 2.2b → "Service charge", 2.2c → "Buildings insurance"
+- Option 2: label each with the full 2.2 question text
+
+**E. §5.1/5.2 decoration years** — schema models `externally`/`internally` containers with a `.year`:
+- 5.1 `externally` → "In what year was the outside of the building last decorated?"
+- 5.2 `internally` → "In what year were any internal communal parts last decorated?"
+
+## Intentionally NOT titled (no distinct printed TA7 label / structural / overlay-only)
+
+Section containers `ownership`, `ownershipsToBeTransferred`, `leaseholdInformation`,
+`ownershipAndManagement`, `requiredDocuments`, `lastDecoratedPeriod`, `contacts`; non-TA7 fields
+`ownershipType`, `isGroundRentPayable`, `typeOfConstruction`; per-line address fields; `13`
+confirmation/declaration block (no numbered form heading); `attachments`/`details` that already
+match the form's "Please supply a copy" / "Please give details".
+
+## Out of scope
+
+Any TA7 questions lacking a `ta7ed5Ref` (ref-completion is separate, per `fix/ta7ed5-*` work).
+
+## Plan after sign-off
+
+Replicate the TA6 mechanism: a `scripts/ta7ed5-titles.js` map keyed by `ref::segment`, surgical
+`jsonc-parser` inserts (after `title`), prefix/colon normalisation incl. existing 20, regenerate
+`ta7ed5.json` via `extractOverlay.js`, run `npm test`, audit all titles vs the form. Branch off `dev`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pdtf/schemas",
-  "version": "3.6.0-dev.7",
+  "version": "3.6.0-dev.8",
   "description": "Property Data Trust Framework Schemas and Utilities",
   "main": "index.js",
   "directories": {