eyeling 1.26.7 → 1.27.1

package/test/examples.test.js

@@ -212,6 +212,8 @@ function runExampleToFile({ root, examplesDir, eyelingJsPath, nodePath, file, ge
 
 function main() {
   const suiteStart = Date.now();
+  const proofOnly = process.argv.includes('--proof-only');
+
 
   // test/examples.test.js -> repo root is one level up
   const root = path.resolve(__dirname, '..');
@@ -230,11 +232,13 @@ function main() {
     process.exit(1);
   }
 
-  const files = fs
-    .readdirSync(examplesDir)
-    .filter((f) => f.endsWith('.n3'))
-    .sort((a, b) => a.localeCompare(b));
-  const proofFiles = fs.existsSync(proofDir)
+  const files = proofOnly
+    ? []
+    : fs
+        .readdirSync(examplesDir)
+        .filter((f) => f.endsWith('.n3'))
+        .sort((a, b) => a.localeCompare(b));
+  const proofFiles = proofOnly && fs.existsSync(proofDir)
     ? fs
         .readdirSync(proofDir)
         .filter((f) => f.endsWith('.n3') && fs.existsSync(path.join(examplesDir, f)))
@@ -242,11 +246,13 @@ function main() {
     : [];
   const totalTests = files.length + proofFiles.length;
 
-  info(`Running ${files.length} examples tests and ${proofFiles.length} proof golden tests`);
+  info(proofOnly
+    ? `Running ${proofFiles.length} proof golden tests`
+    : `Running ${files.length} examples tests`);
   console.log(`${C.dim}${getEyelingVersion(nodePath, eyelingJsPath, root)}; node ${process.version}${C.n}`);
 
-  if (files.length === 0) {
-    ok('No .n3 files found in examples/');
+  if (totalTests === 0) {
+    ok(proofOnly ? 'No proof goldens found in examples/proof/' : 'No .n3 files found in examples/');
     process.exit(0);
   }
 
@@ -256,6 +262,7 @@ function main() {
   // Pretty, stable numbering (e.g., 001..100 when running 100 tests)
   const idxWidth = String(files.length).length;
   const proofIdxWidth = String(Math.max(proofFiles.length, 1)).length;
+  const proofLabel = proofOnly ? '' : 'proof ';
 
   for (let i = 0; i < files.length; i++) {
     const idx = String(i + 1).padStart(idxWidth, '0');
@@ -357,7 +364,7 @@ function main() {
       n3Text = fs.readFileSync(filePath, 'utf8');
     } catch (e) {
       const ms = Date.now() - start;
-      fail(`proof ${idx} ${file} ${msTag(ms)}`);
+      fail(`${proofLabel}${idx} ${file} ${msTag(ms)}`);
       fail(`Cannot read proof input: ${e.message}`);
       failed++;
       continue;
@@ -381,13 +388,13 @@ function main() {
 
     if (diffOk && rcOk) {
       if (expectedRc === 0) {
-        ok(`proof ${idx} ${file} ${msTag(ms)}`);
+        ok(`${proofLabel}${idx} ${file} ${msTag(ms)}`);
       } else {
-        ok(`proof ${idx} ${file} (expected exit ${expectedRc}) ${msTag(ms)}`);
+        ok(`${proofLabel}${idx} ${file} (expected exit ${expectedRc}) ${msTag(ms)}`);
       }
       passed++;
     } else {
-      fail(`proof ${idx} ${file} ${msTag(ms)}`);
+      fail(`${proofLabel}${idx} ${file} ${msTag(ms)}`);
       if (!rcOk) {
         fail(`Exit code ${rc}, expected ${expectedRc}`);
       }
@@ -410,10 +417,14 @@ function main() {
   info(`Total elapsed: ${suiteMs} ms (${(suiteMs / 1000).toFixed(2)} s)`);
 
   if (failed === 0) {
-    ok(`All examples tests passed (${passed}/${totalTests})`);
+    ok(proofOnly
+      ? `All proof golden tests passed (${passed}/${totalTests})`
+      : `All examples tests passed (${passed}/${totalTests})`);
     process.exit(0);
   } else {
-    fail(`Some examples tests failed (${passed}/${totalTests})`);
+    fail(proofOnly
+      ? `Some proof golden tests failed (${passed}/${totalTests})`
+      : `Some examples tests failed (${passed}/${totalTests})`);
     // keep exit code 2 (matches historical behavior of examples/test)
     process.exit(2);
   }

package/test/fixtures/marc-rules-stream-messages.n3

@@ -0,0 +1,192 @@
+# MARC extraction rules for the RDF Messages demo (see marc-sample.messages.nt).
+#
+# Each message is one bibliographic record shaped as a nested list of MARC
+# fields: <record> :record (("245" "1" "0" "a" "Title…") ("650" …) …), where an
+# inner list is (tag ind1 ind2 code value code value …). These backward helper
+# rules (marc:field / marc:subf / marc:map / marc:join / marc:id) walk that
+# structure to pull a subfield's value by (tag, subfield-code); the forward
+# rules at the bottom emit :title / :subject / :type per record.
+#
+
+@prefix : <http://example.org/ns#> .
+@prefix list: <http://www.w3.org/2000/10/swap/list#> .
+@prefix log: <http://www.w3.org/2000/10/swap/log#> .
+@prefix eymsg: <https://eyereasoner.github.io/eyeling/vocab/message#> .
+@prefix string: <http://www.w3.org/2000/10/swap/string#> .
+@prefix math: <http://www.w3.org/2000/10/swap/math#> .
+@prefix marc: <https://codeberg.org/phochste/marcattacks#> .
+
+## ---- Helper functions (backward rules) ----
+
+# marc:splice - drop the first ?Idx members of a list (here: the tag + indicators)
+{ (?List ?Idx) marc:splice ?Result }
+<=
+{
+  ( ?X {
+      ?List list:iterate (?Num ?X).
+      ?Num math:notLessThan ?Idx.
+      } ?Result ) log:collectAllIn _:x.
+}.
+
+# marc:join - join a list with a separator
+{ (?List ?Sep) marc:join ?Result }
+<=
+{
+  (?List ?Sep "") marc:join ?Result.
+}.
+
+{ ( () ?Sep ?Acc ) marc:join ?Acc }
+<= true.
+
+{ ( ?List ?Sep ?Acc ) marc:join ?Result }
+<=
+{
+  ?List list:firstRest (?H ?T).
+  ?Acc log:equalTo "".
+  ( ?T ?Sep ?H ) marc:join ?Result .
+}.
+
+{ ( ?List ?Sep ?Acc ) marc:join ?Result }
+<=
+{
+  ?List list:firstRest (?H ?T).
+  ?Acc log:notEqualTo "".
+  ( ?Acc ?Sep ?H) string:concatenation ?AccNew.
+  ( ?T ?Sep ?AccNew ) marc:join ?Result .
+}.
+
+# marc:id - return the record id as an IRI (from the 001 control field)
+{ ?Record marc:id ?Result }
+<=
+{
+  (?Record "001") marc:field0 ?F001.
+  ?F001 marc:ctrl ?ID.
+  ( "http://lib.ugent.be/record/" ?ID ) string:concatenation ?IRI_ID.
+  ?Result log:uri ?IRI_ID.
+}.
+
+# marc:ctrl - return the control value of a field
+{ ?Field marc:ctrl ?Result }
+<=
+{
+  (?Field 3) list:memberAt "_" .
+  (?Field 4) list:memberAt ?Result.
+}.
+
+# marc:subf - return all values matching a subfield regex
+{ ( ?Field ?Regex) marc:subf ?Result }
+<=
+{
+  ( ?Field 3) marc:splice ?FieldData.
+  ( ?FieldData ?Regex ()) marc:subf ?Result.
+} .
+
+{ ( () ?Regex ?Acc ) marc:subf ?Acc }
+<= true.
+
+{ ( ?FieldData ?Regex ?Acc ) marc:subf ?Result }
+<=
+{
+    ?FieldData list:firstRest (?Subf ?Rest).
+    ?Rest list:firstRest (?Value ?Tail).
+    ?Subf string:matches ?Regex.
+    ( ?Acc (?Value)) list:append ?Acc2.
+    ( ?Tail ?Regex ?Acc2 ) marc:subf ?Result.
+}.
+
+{ ( ?FieldData ?Regex ?Acc ) marc:subf ?Result }
+<=
+{
+    ?FieldData list:firstRest (?Subf ?Rest).
+    ?Rest list:firstRest (?Value ?Tail).
+    ?Subf string:notMatches ?Regex.
+    ( ?Tail ?Regex ?Acc ) marc:subf ?Result.
+}.
+
+# marc:field0 - collect the first row of a marc field
+{ ( ?Record ?Field) marc:field0 ?Result }
+<=
+{
+  ( ?Record ?Field) marc:field ?F.
+  ?F list:first ?Result.
+}.
+
+# marc:field - collect all data for a marc field
+{ ( ?Record ?Field) marc:field ?Result}
+<=
+{
+  ( ?Record ?Field ()) marc:field ?Result.
+}.
+
+{ ( () ?Field ?Acc ) marc:field ?Acc}
+<= true.
+
+{ ( ?L ?Field ?Acc ) marc:field ?Result }
+<=
+{
+  ?L list:firstRest (?H ?T).
+  ( ?H 0 ) list:memberAt ?Field.
+  ( ?Acc (?H) ) list:append ?AccNew.
+  (?T ?Field ?AccNew) marc:field ?Result.
+}.
+
+{ (?L ?Field ?Acc) marc:field ?Result }
+<=
+{
+  ?L list:firstRest (?H ?T).
+  ( ?H 0 ) list:memberAt ?X.
+  ?Field log:notEqualTo ?X.
+  (?T ?Field ?Acc) marc:field ?Result.
+}.
+
+# marc:map - join all values of (tag, subfield) into one string
+{ ( ?Record ?Tag ?Subfield ) marc:map ?Result }
+<=
+{
+  (?Record ?Tag) marc:field ?FL.
+  ?FL list:member ?F.
+  (?F ?Subfield) marc:subf ?T .
+  (?T " ") marc:join ?Result.
+}.
+
+## ---- Extraction rules (forward) ----
+
+# In --stream-messages mode the current RDF Message payload is exposed as a
+# quoted graph via eymsg:payloadGraph/log:nameOf. Match :record inside that
+# payload graph, then reuse the ordinary MARC helper rules on the bound list.
+
+{
+  ?Envelope eymsg:payloadGraph ?Payload.
+  ?Payload log:nameOf ?PayloadContext.
+  ?PayloadContext log:includes { ?X :record ?Record. }.
+  ?Record marc:id ?ID.
+  (?Record "245" "a") marc:map ?Val.
+}
+=>
+{
+  ?ID :title ?Val.
+}.
+
+{
+  ?Envelope eymsg:payloadGraph ?Payload.
+  ?Payload log:nameOf ?PayloadContext.
+  ?PayloadContext log:includes { ?X :record ?Record. }.
+  ?Record marc:id ?ID.
+  (?Record "650" "a") marc:map ?Val.
+}
+=>
+{
+  ?ID :subject ?Val.
+}.
+
+{
+  ?Envelope eymsg:payloadGraph ?Payload.
+  ?Payload log:nameOf ?PayloadContext.
+  ?PayloadContext log:includes { ?X :record ?Record. }.
+  ?Record marc:id ?ID.
+  (?Record "920" "a") marc:map ?Val.
+}
+=>
+{
+  ?ID :type ?Val.
+}.

package/test/stream_messages.test.js

@@ -0,0 +1,284 @@
+'use strict';
+
+const assert = require('node:assert/strict');
+const cp = require('node:child_process');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+
+const root = path.resolve(__dirname, '..');
+const eyelingJsPath = path.join(root, 'eyeling.js');
+
+const TTY = process.stdout.isTTY;
+const C = TTY
+  ? { g: '\x1b[32m', r: '\x1b[31m', y: '\x1b[33m', dim: '\x1b[2m', n: '\x1b[0m' }
+  : { g: '', r: '', y: '', dim: '', n: '' };
+
+function ok(msg) {
+  console.log(`${C.g}OK ${C.n} ${msg}`);
+}
+function info(msg) {
+  console.log(`${C.y}==${C.n} ${msg}`);
+}
+function fail(msg) {
+  console.error(`${C.r}FAIL${C.n} ${msg}`);
+}
+function numberedName(index, name) {
+  return `${String(index + 1).padStart(3, '0')} ${name}`;
+}
+function msNow() {
+  return Date.now();
+}
+function mkTmpDir() {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'eyeling-stream-messages-'));
+}
+function rmrf(p) {
+  try {
+    fs.rmSync(p, { recursive: true, force: true });
+  } catch {}
+}
+
+function runEyeling(args, opts = {}) {
+  return cp.spawnSync(process.execPath, [eyelingJsPath, ...args], {
+    cwd: root,
+    encoding: 'utf8',
+    maxBuffer: opts.maxBuffer || 20 * 1024 * 1024,
+  });
+}
+function expectEyelingOk(args, opts = {}) {
+  const r = runEyeling(args, opts);
+  if (r.status === 0) return r.stdout;
+  throw new Error(
+    `eyeling failed with exit ${r.status}\n` +
+      `STDOUT:\n${r.stdout || ''}\n` +
+      `STDERR:\n${r.stderr || ''}`,
+  );
+}
+
+function writeScopedPayloadRules(file) {
+  fs.writeFileSync(
+    file,
+    `@prefix : <urn:test#>.\n` +
+      `@prefix eymsg: <https://eyereasoner.github.io/eyeling/vocab/message#>.\n` +
+      `@prefix log: <http://www.w3.org/2000/10/swap/log#>.\n` +
+      `{\n` +
+      `  ?Envelope eymsg:payloadGraph ?Payload.\n` +
+      `  ?Payload log:nameOf ?PayloadContext.\n` +
+      `  ?PayloadContext log:includes { ?Subject :line ?Line. }.\n` +
+      `} => {\n` +
+      `  ?Envelope log:outputString ?Line.\n` +
+      `}.\n`,
+    'utf8',
+  );
+}
+function writeBasicMessageLog(file) {
+  fs.writeFileSync(
+    file,
+    `VERSION "1.2-messages"\n` +
+      `PREFIX : <urn:test#>\n` +
+      `\n` +
+      `:a :line "one\\n".\n` +
+      `MESSAGE\n` +
+      `\n` +
+      `:b :line "two\\n".\n` +
+      `MESSAGE\n` +
+      `# empty heartbeat\n` +
+      `MESSAGE\n` +
+      `:c :line "three\\n".\n`,
+    'utf8',
+  );
+}
+function writeLargeMessageLog(file, count) {
+  let text = 'VERSION "1.2-messages"\nPREFIX : <urn:test#>\n';
+  for (let i = 1; i <= count; i += 1) {
+    text += `:m${i} :line "${i}\\n".\n`;
+    if (i < count) text += 'MESSAGE\n';
+  }
+  fs.writeFileSync(file, text, 'utf8');
+}
+function writeMarcMessageLog(file) {
+  fs.writeFileSync(
+    file,
+    `VERSION "1.2-messages"\n` +
+      `PREFIX : <http://example.org/ns#>\n` +
+      `\n` +
+      `:record1 :record (\n` +
+      `  ("001" "_" "_" "_" "42")\n` +
+      `  ("245" "1" "0" "a" "Streaming RDF Messages")\n` +
+      `  ("650" "_" "0" "a" "Semantic Web")\n` +
+      `  ("920" "_" "_" "a" "book")\n` +
+      `).\n` +
+      `MESSAGE\n` +
+      `:record2 :record (\n` +
+      `  ("001" "_" "_" "_" "43")\n` +
+      `  ("245" "1" "0" "a" "Incremental MARC Extraction")\n` +
+      `  ("650" "_" "0" "a" "Linked data")\n` +
+      `  ("920" "_" "_" "a" "article")\n` +
+      `).\n`,
+    'utf8',
+  );
+}
+
+function startFileServer(file) {
+  const dir = path.dirname(file);
+  const script = path.join(dir, 'server.js');
+  const portFile = path.join(dir, 'server.port');
+  fs.writeFileSync(
+    script,
+    `const fs = require('node:fs');\n` +
+      `const http = require('node:http');\n` +
+      `const file = process.argv[2];\n` +
+      `const portFile = process.argv[3];\n` +
+      `const size = fs.statSync(file).size;\n` +
+      `const server = http.createServer((req, res) => {\n` +
+      `  const headers = { 'content-type': 'text/plain', 'accept-ranges': 'bytes' };\n` +
+      `  if (req.method === 'HEAD') { res.writeHead(200, { ...headers, 'content-length': size }); res.end(); return; }\n` +
+      `  const range = req.headers.range;\n` +
+      `  if (range) {\n` +
+      `    const m = /^bytes=(\\d+)-(\\d*)$/.exec(range);\n` +
+      `    const start = m ? Number(m[1]) : 0;\n` +
+      `    const end = m && m[2] ? Math.min(Number(m[2]), size - 1) : size - 1;\n` +
+      `    res.writeHead(206, { ...headers, 'content-range': 'bytes ' + start + '-' + end + '/' + size, 'content-length': end - start + 1 });\n` +
+      `    fs.createReadStream(file, { start, end }).pipe(res);\n` +
+      `    return;\n` +
+      `  }\n` +
+      `  res.writeHead(200, { ...headers, 'content-length': size });\n` +
+      `  fs.createReadStream(file).pipe(res);\n` +
+      `});\n` +
+      `server.listen(0, '127.0.0.1', () => fs.writeFileSync(portFile, String(server.address().port)));\n`,
+    'utf8',
+  );
+  const child = cp.spawn(process.execPath, [script, file, portFile], { cwd: root, stdio: ['ignore', 'ignore', 'pipe'] });
+  const deadline = Date.now() + 5000;
+  while (Date.now() < deadline) {
+    if (fs.existsSync(portFile)) {
+      const port = fs.readFileSync(portFile, 'utf8').trim();
+      return { url: `http://127.0.0.1:${port}/messages.txt`, stop: () => child.kill() };
+    }
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);
+  }
+  child.kill();
+  throw new Error('test HTTP server did not start');
+}
+
+const cases = [
+  {
+    name: 'scoped payload rules run once per RDF Message',
+    run(tmp) {
+      const rules = path.join(tmp, 'rules.n3');
+      const log = path.join(tmp, 'messages.trig');
+      writeScopedPayloadRules(rules);
+      writeBasicMessageLog(log);
+      const out = expectEyelingOk(['-r', '--stream-messages', rules, log]);
+      assert.equal(out, 'one\ntwo\nthree\n');
+    },
+  },
+  {
+    name: 'empty heartbeat messages do not leak previous payloads',
+    run(tmp) {
+      const rules = path.join(tmp, 'rules.n3');
+      const log = path.join(tmp, 'messages.trig');
+      writeScopedPayloadRules(rules);
+      writeBasicMessageLog(log);
+      const out = expectEyelingOk(['-r', '--stream-messages', rules, log]);
+      assert.equal(out.trim().split('\n').length, 3);
+      assert.ok(!out.includes('one\none'));
+    },
+  },
+  {
+    name: 'large message logs stream without requiring one monolithic dataset',
+    run(tmp) {
+      const rules = path.join(tmp, 'rules.n3');
+      const log = path.join(tmp, 'large.trig');
+      writeScopedPayloadRules(rules);
+      writeLargeMessageLog(log, 1000);
+      const out = expectEyelingOk(['-r', '--stream-messages', rules, log]);
+      const lines = out.trim().split('\n');
+      assert.equal(lines.length, 1000);
+      assert.equal(new Set(lines).size, 1000);
+      assert.ok(lines.includes('1'));
+      assert.ok(lines.includes('999'));
+      assert.ok(lines.includes('1000'));
+    },
+  },
+  {
+    name: 'remote text/plain RDF Message Logs are streamed via HTTP',
+    run(tmp) {
+      const rules = path.join(tmp, 'rules.n3');
+      const log = path.join(tmp, 'remote.trig');
+      writeScopedPayloadRules(rules);
+      writeLargeMessageLog(log, 25);
+      const server = startFileServer(log);
+      try {
+        const out = expectEyelingOk(['-r', '--stream-messages', server.url, rules]);
+        const lines = out.trim().split('\n');
+        assert.equal(lines.length, 25);
+        assert.equal(lines[0], '1');
+        assert.equal(lines[24], '25');
+      } finally {
+        server.stop();
+      }
+    },
+  },
+  {
+    name: 'MARC extraction rules fire over each streamed payload graph',
+    run(tmp) {
+      const log = path.join(tmp, 'marc.messages.txt');
+      writeMarcMessageLog(log);
+      const fixture = path.join(root, 'test', 'fixtures', 'marc-rules-stream-messages.n3');
+      const out = expectEyelingOk(['-r', '--stream-messages', fixture, log]);
+      assert.deepEqual(out.trim().split('\n').sort(), [
+        '<http://lib.ugent.be/record/42> <http://example.org/ns#subject> "Semantic Web" .',
+        '<http://lib.ugent.be/record/42> <http://example.org/ns#title> "Streaming RDF Messages" .',
+        '<http://lib.ugent.be/record/42> <http://example.org/ns#type> "book" .',
+        '<http://lib.ugent.be/record/43> <http://example.org/ns#subject> "Linked data" .',
+        '<http://lib.ugent.be/record/43> <http://example.org/ns#title> "Incremental MARC Extraction" .',
+        '<http://lib.ugent.be/record/43> <http://example.org/ns#type> "article" .',
+      ]);
+    },
+  },
+  {
+    name: '--stream-messages requires RDF mode',
+    run(tmp) {
+      const rules = path.join(tmp, 'rules.n3');
+      const log = path.join(tmp, 'messages.trig');
+      writeScopedPayloadRules(rules);
+      writeBasicMessageLog(log);
+      const r = runEyeling(['--stream-messages', rules, log]);
+      assert.notEqual(r.status, 0);
+      assert.match(r.stderr, /requires -r\/--rdf/);
+    },
+  },
+];
+
+(function main() {
+  const suiteStart = msNow();
+  info(`Running ${cases.length} stream-message tests`);
+  let passed = 0;
+  let failed = 0;
+  for (const [index, tc] of cases.entries()) {
+    const tmp = mkTmpDir();
+    const testName = numberedName(index, tc.name);
+    const start = msNow();
+    try {
+      tc.run(tmp);
+      ok(`${testName} ${C.dim}(${msNow() - start} ms)${C.n}`);
+      passed++;
+    } catch (e) {
+      fail(`${testName} ${C.dim}(${msNow() - start} ms)${C.n}`);
+      fail(e && e.stack ? e.stack : String(e));
+      failed++;
+    } finally {
+      rmrf(tmp);
+    }
+  }
+  console.log('');
+  const suiteMs = msNow() - suiteStart;
+  console.log(`${C.y}==${C.n} Total elapsed: ${suiteMs} ms (${(suiteMs / 1000).toFixed(2)} s)`);
+  if (failed === 0) {
+    ok(`All stream-message tests passed (${passed}/${cases.length})`);
+    process.exit(0);
+  }
+  fail(`Some stream-message tests failed (${passed}/${cases.length})`);
+  process.exit(1);
+})();