@nuggetslife/vc 0.0.29 → 0.1.0

package/test_w3c_conformance.mjs

@@ -0,0 +1,419 @@
+// W3C JSON-LD 1.1 API + RDF Dataset Canonicalization (RDFC-1.0) conformance
+// runner for the clientffi NAPI JsonLd binding.
+//
+// Two modes:
+//   node test_w3c_conformance.mjs --check    (CI default)
+//     run all suites, compare against w3c-baseline.json, exit non-zero
+//     if any test that previously passed now fails (a regression).
+//
+//   node test_w3c_conformance.mjs --update
+//     run all suites, overwrite w3c-baseline.json with the current
+//     pass set. Use when intentionally widening conformance.
+//
+// Without flags, runs and prints a summary table without baseline comparison.
+//
+// Other flags:
+//   --only <algo>   limit to one of: expand|compact|flatten|toRdf|fromRdf|canonize
+//   --verbose       print PASS/FAIL/ERR per test
+
+import { readFileSync, writeFileSync, existsSync, readdirSync, statSync } from 'node:fs';
+import { resolve, dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { JsonLd } from './index.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const args = process.argv.slice(2);
+const flag = (n) => args.includes(n);
+const valOf = (n) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : null; };
+const verbose = flag('--verbose');
+const onlyArg = valOf('--only');
+const checkBaseline = flag('--check');
+const updateBaseline = flag('--update');
+
+const TESTS_ROOT = resolve(__dirname, 'tmp-w3c-tests');
+const JLD_TESTS = join(TESTS_ROOT, 'json-ld-api', 'tests');
+const JLD_FRAMING_TESTS = join(TESTS_ROOT, 'json-ld-framing', 'tests');
+const RDFC_TESTS = join(TESTS_ROOT, 'rdf-canon', 'tests');
+const JLD_BASE = 'https://w3c.github.io/json-ld-api/tests/';
+const BASELINE_PATH = join(__dirname, 'w3c-baseline.json');
+
+// Per-test wall-clock cap. clientffi has known hangs in a few hot paths;
+// without this they freeze the runner.
+const TIMEOUT_MS = 30000;
+const PROGRESS_EVERY = 25; // emit a heartbeat every N tests so CI's
+                            // no_output_timeout doesn't kill the runner
+
+function withTimeout(promiseOrValue, label) {
+    // proc.fromRDF returns synchronously; everything else is async.
+    const p = Promise.resolve(promiseOrValue);
+    let t;
+    const timeoutPromise = new Promise((_, reject) => {
+        t = setTimeout(() => reject(new Error('timeout: ' + label)), TIMEOUT_MS);
+    });
+    // If the main op settles first, the timeout-rejection still fires later;
+    // consume that specific rejection rather than suppressing process-wide.
+    timeoutPromise.catch(() => {});
+    return Promise.race([p.finally(() => clearTimeout(t)), timeoutPromise]);
+}
+
+const loadJson = (p) => JSON.parse(readFileSync(p, 'utf8'));
+const loadText = (p) => readFileSync(p, 'utf8');
+
+function preloadJldContexts() {
+    const ctxs = {};
+    function walk(dir, prefix) {
+        for (const name of readdirSync(dir)) {
+            const full = join(dir, name);
+            if (statSync(full).isDirectory()) walk(full, prefix + name + '/');
+            else if (name.endsWith('.jsonld') || name.endsWith('.json')) {
+                try { ctxs[JLD_BASE + prefix + name] = loadJson(full); } catch {}
+            }
+        }
+    }
+    walk(JLD_TESTS, '');
+    if (existsSync(JLD_FRAMING_TESTS)) walk(JLD_FRAMING_TESTS, '');
+    return ctxs;
+}
+
+function canonicalStringify(v) {
+    if (v === null || typeof v !== 'object') return JSON.stringify(v);
+    if (Array.isArray(v)) return '[' + v.map(canonicalStringify).join(',') + ']';
+    const keys = Object.keys(v).sort();
+    return '{' + keys.map(k => JSON.stringify(k) + ':' + canonicalStringify(v[k])).join(',') + '}';
+}
+const jsonEqual = (a, b) => canonicalStringify(a) === canonicalStringify(b);
+const normNQuads = (s) => typeof s === 'string'
+    ? s.split('\n').map(l => l.trim()).filter(Boolean).sort().join('\n')
+    : s;
+
+// W3C toRdf expected fixtures contain post-URDNA2015 blank-node labels
+// (e.g. `_:c14n0`), but the spec itself defines toRdf as producing
+// pre-canonical labels (e.g. `_:b0`). jsonld.js exhibits the same
+// pre-canonical output. To compare semantically we run URDNA2015 on both
+// sides before string-matching. Falls back to normNQuads if either side
+// can't be canonicalized (so a bug in the processor still surfaces).
+const canonProc = new JsonLd();
+async function nquadsEqual(got, expected) {
+    if (typeof got !== 'string' || typeof expected !== 'string') return false;
+    try {
+        const opts = { algorithm: 'URDNA2015', inputFormat: 'application/n-quads', format: 'application/n-quads' };
+        const a = await canonProc.canonize(got, opts);
+        const b = await canonProc.canonize(expected, opts);
+        return a === b;
+    } catch {
+        return normNQuads(got) === normNQuads(expected);
+    }
+}
+
+const DENYLIST_PATH = join(__dirname, 'w3c-denylist.json');
+let DENYLIST = {};
+if (existsSync(DENYLIST_PATH)) {
+    DENYLIST = loadJson(DENYLIST_PATH);
+}
+const isDenied = (algo, id) => (DENYLIST[algo] || []).includes(id);
+
+// In --check mode, restrict the run to baseline-passing tests only — the
+// gate's purpose is regression detection, not full re-evaluation. Cuts
+// CI runtime ~70% (skips ~450 baseline-failing tests).
+let CHECK_ALLOW = null;
+if (checkBaseline && existsSync(BASELINE_PATH)) {
+    const b = loadJson(BASELINE_PATH);
+    CHECK_ALLOW = {};
+    for (const [algo, r] of Object.entries(b)) {
+        CHECK_ALLOW[algo] = new Set(r.passing || []);
+    }
+}
+const isInCheckScope = (algo, id) =>
+    !CHECK_ALLOW || (CHECK_ALLOW[algo] && CHECK_ALLOW[algo].has(id));
+
+const RESULTS = {}; // { algo: { passing: Set, failing: Set, erroring: Set } }
+function record(algo, id, status) {
+    const r = RESULTS[algo] ||= { passing: new Set(), failing: new Set(), erroring: new Set() };
+    if (status === 'pass') r.passing.add(id);
+    else if (status === 'fail') r.failing.add(id);
+    else r.erroring.add(id);
+    if (verbose) {
+        const tag = { pass: 'PASS', fail: 'FAIL', err: 'ERR ' }[status];
+        process.stderr.write(`[${tag}] ${algo} ${id}\n`);
+    }
+}
+
+async function runJld(algo, methodName) {
+    const manifestPath = join(JLD_TESTS, `${algo}-manifest.jsonld`);
+    if (!existsSync(manifestPath)) return;
+    const manifest = loadJson(manifestPath);
+    const contexts = preloadJldContexts();
+    const proc = new JsonLd({ contexts });
+    let seen = 0;
+
+    for (const entry of manifest.sequence) {
+        const types = Array.isArray(entry['@type']) ? entry['@type'] : [entry['@type']];
+        const isPositive = types.includes('jld:PositiveEvaluationTest');
+        const isNegative = types.includes('jld:NegativeEvaluationTest');
+        const id = entry['@id'];
+        if (!isPositive && !isNegative) continue;
+        if (isDenied(algo, id)) continue;
+        if (!isInCheckScope(algo, id)) continue;
+        seen += 1;
+        if (seen % PROGRESS_EVERY === 0) {
+            process.stderr.write(`  ${algo}: ${seen} tests processed\n`);
+        }
+        const inputUrl = JLD_BASE + entry.input;
+        const inputPath = join(JLD_TESTS, entry.input);
+        if (!existsSync(inputPath)) { record(algo, id, 'err'); continue; }
+
+        const opt = entry.option || {};
+        const callOpts = { base: opt.base || inputUrl };
+        if (opt.processingMode) callOpts.processingMode = opt.processingMode;
+        else if (opt.specVersion) callOpts.processingMode = opt.specVersion;
+        if (opt.expandContext) {
+            const ctxUrl = JLD_BASE + opt.expandContext;
+            callOpts.expandContext = contexts[ctxUrl] || ctxUrl;
+        }
+        // toRdf-specific options from W3C test entries
+        if (opt.rdfDirection) callOpts.rdfDirection = opt.rdfDirection;
+        if (opt.produceGeneralizedRdf != null) callOpts.produceGeneralizedRdf = opt.produceGeneralizedRdf;
+        if (opt.compactArrays != null) callOpts.compactArrays = opt.compactArrays;
+        // fromRDF-specific options
+        if (opt.useNativeTypes != null) callOpts.useNativeTypes = opt.useNativeTypes;
+        if (opt.useRdfType != null) callOpts.useRdfType = opt.useRdfType;
+
+        try {
+            const input = methodName === 'fromRDF' ? loadText(inputPath) : loadJson(inputPath);
+            let napiResult;
+            if (methodName === 'expand') napiResult = await withTimeout(proc.expand(input, callOpts), id);
+            else if (methodName === 'compact') {
+                const ctxUrl = JLD_BASE + entry.context;
+                const ctx = contexts[ctxUrl] || ctxUrl;
+                napiResult = await withTimeout(proc.compact(input, ctx, callOpts), id);
+            } else if (methodName === 'flatten') {
+                const ctxUrl = entry.context ? (JLD_BASE + entry.context) : null;
+                const ctx = ctxUrl ? (contexts[ctxUrl] || ctxUrl) : null;
+                napiResult = await withTimeout(proc.flatten(input, ctx, callOpts), id);
+            } else if (methodName === 'toRDF') {
+                napiResult = await withTimeout(proc.toRDF(input, { ...callOpts, format: 'application/n-quads' }), id);
+            } else if (methodName === 'fromRDF') napiResult = await withTimeout(proc.fromRDF(input, callOpts), id);
+
+            if (isNegative) { record(algo, id, 'fail'); continue; }
+
+            const expectPath = join(JLD_TESTS, entry.expect);
+            if (!existsSync(expectPath)) { record(algo, id, 'err'); continue; }
+            const expected = methodName === 'toRDF' ? loadText(expectPath) : loadJson(expectPath);
+
+            const ok = methodName === 'toRDF'
+                ? await nquadsEqual(napiResult, expected)
+                : jsonEqual(napiResult, expected);
+            record(algo, id, ok ? 'pass' : 'fail');
+        } catch (e) {
+            // Negative tests expect a processor error; only count as pass if it
+            // genuinely came from the processor, not from a timeout or harness bug.
+            const isTimeout = e && /^timeout:/.test(e.message || '');
+            record(algo, id, isNegative && !isTimeout ? 'pass' : 'err');
+        }
+    }
+}
+
+async function runRdfc() {
+    const manifestPath = join(RDFC_TESTS, 'manifest.jsonld');
+    if (!existsSync(manifestPath)) return;
+    const manifest = loadJson(manifestPath);
+    const algo = 'canonize';
+    const proc = new JsonLd();
+    let seen = 0;
+
+    for (const entry of manifest.entries) {
+        if (entry.type !== 'rdfc:RDFC10EvalTest') continue;
+        const id = entry.id;
+        if (isDenied('canonize', id)) continue;
+        if (!isInCheckScope('canonize', id)) continue;
+        seen += 1;
+        if (seen % PROGRESS_EVERY === 0) {
+            process.stderr.write(`  canonize: ${seen} tests processed\n`);
+        }
+        const inputPath = join(RDFC_TESTS, entry.action);
+        const expectedPath = join(RDFC_TESTS, entry.result);
+        if (!existsSync(inputPath) || !existsSync(expectedPath)) { record(algo, id, 'err'); continue; }
+        try {
+            const input = loadText(inputPath);
+            const expected = loadText(expectedPath);
+            const opts = {
+                inputFormat: 'application/n-quads',
+                format: 'application/n-quads',
+                algorithm: 'RDFC-1.0',
+            };
+            if (entry.hashAlgorithm) opts.hashAlgorithm = entry.hashAlgorithm;
+            const got = await withTimeout(proc.canonize(input, opts), id);
+            record(algo, id, got === expected ? 'pass' : 'fail');
+        } catch (e) {
+            record(algo, id, 'err');
+        }
+    }
+}
+
+async function runOnce(algo) {
+    process.stderr.write(`Running ${algo}...\n`);
+    if (algo === 'canonize') await runRdfc();
+    else if (algo === 'expand') await runJld('expand', 'expand');
+    else if (algo === 'compact') await runJld('compact', 'compact');
+    else if (algo === 'flatten') await runJld('flatten', 'flatten');
+    else if (algo === 'toRdf') await runJld('toRdf', 'toRDF');
+    else if (algo === 'fromRdf') await runJld('fromRdf', 'fromRDF');
+    else if (algo === 'frame') await runFraming();
+}
+
+async function runFraming() {
+    const manifestPath = join(JLD_FRAMING_TESTS, 'frame-manifest.jsonld');
+    if (!existsSync(manifestPath)) return;
+    const manifest = loadJson(manifestPath);
+    const contexts = preloadJldContexts();
+    const proc = new JsonLd({ contexts });
+    let seen = 0;
+    for (const entry of manifest.sequence) {
+        const types = Array.isArray(entry['@type']) ? entry['@type'] : [entry['@type']];
+        if (!types.includes('jld:PositiveEvaluationTest') && !types.includes('jld:NegativeEvaluationTest')) continue;
+        const id = entry['@id'];
+        if (isDenied('frame', id)) continue;
+        if (!isInCheckScope('frame', id)) continue;
+        seen += 1;
+        if (seen % PROGRESS_EVERY === 0) process.stderr.write(`  frame: ${seen} tests processed\n`);
+        if (!entry.input || !entry.frame) { record('frame', id, 'err'); continue; }
+        const inputPath = join(JLD_FRAMING_TESTS, entry.input);
+        const framePath = join(JLD_FRAMING_TESTS, entry.frame);
+        const expectPath = entry.expect ? join(JLD_FRAMING_TESTS, entry.expect) : null;
+        if (!existsSync(inputPath) || !existsSync(framePath)) { record('frame', id, 'err'); continue; }
+        const opt = entry.option || {};
+        const inputUrl = 'https://w3c.github.io/json-ld-framing/tests/' + entry.input;
+        const callOpts = { base: opt.base || inputUrl };
+        if (opt.processingMode) callOpts.processingMode = opt.processingMode;
+        else if (opt.specVersion) callOpts.processingMode = opt.specVersion;
+        if (opt.omitGraph != null) callOpts.omitGraph = opt.omitGraph;
+        if (opt.requireAll != null) callOpts.requireAll = opt.requireAll;
+        try {
+            const input = loadJson(inputPath);
+            const frame = loadJson(framePath);
+            const napiResult = await withTimeout(proc.frame(input, frame, callOpts), id);
+            if (!expectPath || !existsSync(expectPath)) { record('frame', id, 'err'); continue; }
+            const expected = loadJson(expectPath);
+            record('frame', id, jsonEqual(napiResult, expected) ? 'pass' : 'fail');
+        } catch (e) {
+            const isTimeout = e && /^timeout:/.test(e.message || '');
+            record('frame', id, isTimeout ? 'err' : 'err');
+        }
+    }
+}
+
+
+async function main() {
+    if (!existsSync(JLD_TESTS) || !existsSync(RDFC_TESTS)) {
+        console.error('W3C test suites not found at ' + TESTS_ROOT);
+        console.error('Run: vc/js/scripts/fetch-w3c-tests.sh');
+        process.exit(2);
+    }
+    // flatten and toRdf are gated by V2: the v1 path has Rust-level deadlocks
+    // (sync block_on bridging into tokio::sync locks in the document loader and
+    // node_map). The v2 path (jsonld_v2::flatten_v2 / to_nquads_v2) uses a
+    // pre-resolved HashMap loader with no tokio sync primitives and runs the
+    // suite without hangs, so it is included in the default list.
+    const algos = ['canonize', 'fromRdf', 'expand', 'compact', 'flatten', 'toRdf', 'frame'];
+    const allAlgos = ['expand', 'compact', 'flatten', 'toRdf', 'fromRdf', 'canonize', 'frame'];
+    if (onlyArg && !allAlgos.includes(onlyArg)) {
+        console.error(`Invalid --only value: ${onlyArg}. Expected one of: ${allAlgos.join(', ')}`);
+        process.exit(2);
+    }
+    const filtered = onlyArg ? [onlyArg] : algos;
+
+    // --runs N: run each algo N times and intersect the passing sets so
+    // flaky tests don't end up in the baseline. Default 1.
+    // Note: slow algos (expand, compact) hang on N>1 because clientffi's
+    // NAPI worker pool fills with hung threads from earlier passes. Use
+    // --runs 3 only for fast algos like canonize/fromRdf.
+    const runsArg = valOf('--runs');
+    const repeats = runsArg == null ? 1 : Number.parseInt(runsArg, 10);
+    if (!Number.isInteger(repeats) || repeats <= 0 || String(repeats) !== runsArg) {
+        if (runsArg != null) {
+            console.error(`Invalid value for --runs: ${runsArg}. Expected a positive integer.`);
+            process.exit(2);
+        }
+    }
+    const passingPerRun = []; // array of { algo: Set<id> }, length = repeats
+
+    for (let r = 0; r < repeats; r++) {
+        if (repeats > 1) process.stderr.write(`-- pass ${r + 1}/${repeats} --\n`);
+        // Fresh RESULTS for this pass
+        for (const k of Object.keys(RESULTS)) delete RESULTS[k];
+        for (const algo of filtered) await runOnce(algo);
+        const snapshot = {};
+        for (const [algo, r2] of Object.entries(RESULTS)) snapshot[algo] = new Set(r2.passing);
+        passingPerRun.push(snapshot);
+    }
+
+    // Build current state. For --update with repeats=3, "passing" is the
+    // intersection across all runs. For --check (single run), use the last run.
+    const current = {};
+    for (const [algo, r] of Object.entries(RESULTS)) {
+        let passing;
+        if (repeats > 1) {
+            passing = [...passingPerRun[0][algo] || []].filter(id =>
+                passingPerRun.every(snap => snap[algo]?.has(id))
+            );
+        } else {
+            passing = [...r.passing];
+        }
+        current[algo] = {
+            passing: passing.sort(),
+            failing: [...r.failing].sort(),
+            erroring: [...r.erroring].sort(),
+        };
+    }
+
+    // Print summary
+    console.log('\n========== W3C CONFORMANCE ==========');
+    console.log(['Algo'.padEnd(10), 'Total'.padEnd(8), 'Pass'.padEnd(8), 'Fail'.padEnd(8), 'Error'.padEnd(8), 'Pass%'].join(''));
+    for (const [algo, r] of Object.entries(current)) {
+        const total = r.passing.length + r.failing.length + r.erroring.length;
+        const pct = total ? ((r.passing.length / total) * 100).toFixed(1) + '%' : '-';
+        console.log([algo.padEnd(10), String(total).padEnd(8), String(r.passing.length).padEnd(8), String(r.failing.length).padEnd(8), String(r.erroring.length).padEnd(8), pct].join(''));
+    }
+
+    if (updateBaseline) {
+        // Merge with existing baseline so single-algo runs accumulate.
+        const merged = existsSync(BASELINE_PATH) ? loadJson(BASELINE_PATH) : {};
+        for (const [algo, r] of Object.entries(current)) merged[algo] = r;
+        writeFileSync(BASELINE_PATH, JSON.stringify(merged, null, 2) + '\n');
+        console.log(`\nBaseline ${onlyArg ? 'updated for ' + onlyArg : 'written'} at ${BASELINE_PATH}`);
+        return;
+    }
+
+    if (checkBaseline) {
+        if (!existsSync(BASELINE_PATH)) {
+            console.error(`\nNo baseline at ${BASELINE_PATH}. Run with --update to create one.`);
+            process.exit(2);
+        }
+        const baseline = loadJson(BASELINE_PATH);
+        const regressions = [];
+        const improvements = [];
+        const algosToCheck = onlyArg ? [onlyArg] : Object.keys(baseline);
+        for (const algo of algosToCheck) {
+            if (!baseline[algo]) continue;
+            const wasPassing = new Set(baseline[algo].passing);
+            const nowPassing = new Set(current[algo]?.passing || []);
+            for (const id of wasPassing) if (!nowPassing.has(id)) regressions.push({ algo, id });
+            for (const id of nowPassing) if (!wasPassing.has(id)) improvements.push({ algo, id });
+        }
+
+        if (improvements.length) {
+            console.log(`\n${improvements.length} test(s) newly passing — please run --update to ratchet the baseline:`);
+            for (const { algo, id } of improvements.slice(0, 20)) console.log(`  + ${algo} ${id}`);
+            if (improvements.length > 20) console.log(`  ... ${improvements.length - 20} more`);
+        }
+
+        if (regressions.length) {
+            console.log(`\n${regressions.length} REGRESSION(S) — these tests passed in baseline but now fail:`);
+            for (const { algo, id } of regressions) console.log(`  - ${algo} ${id}`);
+            process.exit(1);
+        }
+        console.log('\nNo regressions vs baseline.');
+    }
+}
+
+main().catch(e => { console.error(e); process.exit(1); });