eyeling 1.26.7 → 1.27.0

package/test/fixtures/marc-rules-stream-messages.n3

@@ -0,0 +1,192 @@
+# MARC extraction rules for the RDF Messages demo (see marc-sample.messages.nt).
+#
+# Each message is one bibliographic record shaped as a nested list of MARC
+# fields: <record> :record (("245" "1" "0" "a" "Title…") ("650" …) …), where an
+# inner list is (tag ind1 ind2 code value code value …). These backward helper
+# rules (marc:field / marc:subf / marc:map / marc:join / marc:id) walk that
+# structure to pull a subfield's value by (tag, subfield-code); the forward
+# rules at the bottom emit :title / :subject / :type per record.
+#
+
+@prefix : <http://example.org/ns#> .
+@prefix list: <http://www.w3.org/2000/10/swap/list#> .
+@prefix log: <http://www.w3.org/2000/10/swap/log#> .
+@prefix eymsg: <https://eyereasoner.github.io/eyeling/vocab/message#> .
+@prefix string: <http://www.w3.org/2000/10/swap/string#> .
+@prefix math: <http://www.w3.org/2000/10/swap/math#> .
+@prefix marc: <https://codeberg.org/phochste/marcattacks#> .
+
+## ---- Helper functions (backward rules) ----
+
+# marc:splice - drop the first ?Idx members of a list (here: the tag + indicators)
+{ (?List ?Idx) marc:splice ?Result }
+<=
+{
+  ( ?X {
+      ?List list:iterate (?Num ?X).
+      ?Num math:notLessThan ?Idx.
+      } ?Result ) log:collectAllIn _:x.
+}.
+
+# marc:join - join a list with a separator
+{ (?List ?Sep) marc:join ?Result }
+<=
+{
+  (?List ?Sep "") marc:join ?Result.
+}.
+
+{ ( () ?Sep ?Acc ) marc:join ?Acc }
+<= true.
+
+{ ( ?List ?Sep ?Acc ) marc:join ?Result }
+<=
+{
+  ?List list:firstRest (?H ?T).
+  ?Acc log:equalTo "".
+  ( ?T ?Sep ?H ) marc:join ?Result .
+}.
+
+{ ( ?List ?Sep ?Acc ) marc:join ?Result }
+<=
+{
+  ?List list:firstRest (?H ?T).
+  ?Acc log:notEqualTo "".
+  ( ?Acc ?Sep ?H) string:concatenation ?AccNew.
+  ( ?T ?Sep ?AccNew ) marc:join ?Result .
+}.
+
+# marc:id - return the record id as an IRI (from the 001 control field)
+{ ?Record marc:id ?Result }
+<=
+{
+  (?Record "001") marc:field0 ?F001.
+  ?F001 marc:ctrl ?ID.
+  ( "http://lib.ugent.be/record/" ?ID ) string:concatenation ?IRI_ID.
+  ?Result log:uri ?IRI_ID.
+}.
+
+# marc:ctrl - return the control value of a field
+{ ?Field marc:ctrl ?Result }
+<=
+{
+  (?Field 3) list:memberAt "_" .
+  (?Field 4) list:memberAt ?Result.
+}.
+
+# marc:subf - return all values matching a subfield regex
+{ ( ?Field ?Regex) marc:subf ?Result }
+<=
+{
+  ( ?Field 3) marc:splice ?FieldData.
+  ( ?FieldData ?Regex ()) marc:subf ?Result.
+} .
+
+{ ( () ?Regex ?Acc ) marc:subf ?Acc }
+<= true.
+
+{ ( ?FieldData ?Regex ?Acc ) marc:subf ?Result }
+<=
+{
+    ?FieldData list:firstRest (?Subf ?Rest).
+    ?Rest list:firstRest (?Value ?Tail).
+    ?Subf string:matches ?Regex.
+    ( ?Acc (?Value)) list:append ?Acc2.
+    ( ?Tail ?Regex ?Acc2 ) marc:subf ?Result.
+}.
+
+{ ( ?FieldData ?Regex ?Acc ) marc:subf ?Result }
+<=
+{
+    ?FieldData list:firstRest (?Subf ?Rest).
+    ?Rest list:firstRest (?Value ?Tail).
+    ?Subf string:notMatches ?Regex.
+    ( ?Tail ?Regex ?Acc ) marc:subf ?Result.
+}.
+
+# marc:field0 - collect the first row of a marc field
+{ ( ?Record ?Field) marc:field0 ?Result }
+<=
+{
+  ( ?Record ?Field) marc:field ?F.
+  ?F list:first ?Result.
+}.
+
+# marc:field - collect all data for a marc field
+{ ( ?Record ?Field) marc:field ?Result}
+<=
+{
+  ( ?Record ?Field ()) marc:field ?Result.
+}.
+
+{ ( () ?Field ?Acc ) marc:field ?Acc}
+<= true.
+
+{ ( ?L ?Field ?Acc ) marc:field ?Result }
+<=
+{
+  ?L list:firstRest (?H ?T).
+  ( ?H 0 ) list:memberAt ?Field.
+  ( ?Acc (?H) ) list:append ?AccNew.
+  (?T ?Field ?AccNew) marc:field ?Result.
+}.
+
+{ (?L ?Field ?Acc) marc:field ?Result }
+<=
+{
+  ?L list:firstRest (?H ?T).
+  ( ?H 0 ) list:memberAt ?X.
+  ?Field log:notEqualTo ?X.
+  (?T ?Field ?Acc) marc:field ?Result.
+}.
+
+# marc:map - join all values of (tag, subfield) into one string
+{ ( ?Record ?Tag ?Subfield ) marc:map ?Result }
+<=
+{
+  (?Record ?Tag) marc:field ?FL.
+  ?FL list:member ?F.
+  (?F ?Subfield) marc:subf ?T .
+  (?T " ") marc:join ?Result.
+}.
+
+## ---- Extraction rules (forward) ----
+
+# In --stream-messages mode the current RDF Message payload is exposed as a
+# quoted graph via eymsg:payloadGraph/log:nameOf. Match :record inside that
+# payload graph, then reuse the ordinary MARC helper rules on the bound list.
+
+{
+  ?Envelope eymsg:payloadGraph ?Payload.
+  ?Payload log:nameOf ?PayloadContext.
+  ?PayloadContext log:includes { ?X :record ?Record. }.
+  ?Record marc:id ?ID.
+  (?Record "245" "a") marc:map ?Val.
+}
+=>
+{
+  ?ID :title ?Val.
+}.
+
+{
+  ?Envelope eymsg:payloadGraph ?Payload.
+  ?Payload log:nameOf ?PayloadContext.
+  ?PayloadContext log:includes { ?X :record ?Record. }.
+  ?Record marc:id ?ID.
+  (?Record "650" "a") marc:map ?Val.
+}
+=>
+{
+  ?ID :subject ?Val.
+}.
+
+{
+  ?Envelope eymsg:payloadGraph ?Payload.
+  ?Payload log:nameOf ?PayloadContext.
+  ?PayloadContext log:includes { ?X :record ?Record. }.
+  ?Record marc:id ?ID.
+  (?Record "920" "a") marc:map ?Val.
+}
+=>
+{
+  ?ID :type ?Val.
+}.

package/test/stream_messages.test.js

@@ -0,0 +1,243 @@
+'use strict';
+
+const assert = require('node:assert/strict');
+const cp = require('node:child_process');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+
+const root = path.resolve(__dirname, '..');
+const eyelingJsPath = path.join(root, 'eyeling.js');
+
+const TTY = process.stdout.isTTY;
+const C = TTY
+  ? { g: '\x1b[32m', r: '\x1b[31m', y: '\x1b[33m', dim: '\x1b[2m', n: '\x1b[0m' }
+  : { g: '', r: '', y: '', dim: '', n: '' };
+
+function ok(msg) {
+  console.log(`${C.g}OK ${C.n} ${msg}`);
+}
+function info(msg) {
+  console.log(`${C.y}==${C.n} ${msg}`);
+}
+function fail(msg) {
+  console.error(`${C.r}FAIL${C.n} ${msg}`);
+}
+
+function numberedName(index, name) {
+  return `${String(index + 1).padStart(3, '0')} ${name}`;
+}
+
+function msNow() {
+  return Date.now();
+}
+
+function mkTmpDir() {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'eyeling-stream-messages-'));
+}
+
+function rmrf(p) {
+  try {
+    fs.rmSync(p, { recursive: true, force: true });
+  } catch {}
+}
+
+function runEyeling(args, opts = {}) {
+  return cp.spawnSync(process.execPath, [eyelingJsPath, ...args], {
+    cwd: root,
+    encoding: 'utf8',
+    maxBuffer: opts.maxBuffer || 20 * 1024 * 1024,
+  });
+}
+
+function expectEyelingOk(args, opts = {}) {
+  const r = runEyeling(args, opts);
+  if (r.status === 0) return r.stdout;
+
+  throw new Error(
+    `eyeling failed with exit ${r.status}\n` +
+      `STDOUT:\n${r.stdout || ''}\n` +
+      `STDERR:\n${r.stderr || ''}`,
+  );
+}
+
+function writeScopedPayloadRules(file) {
+  fs.writeFileSync(
+    file,
+    `@prefix : <urn:test#>.\n` +
+      `@prefix eymsg: <https://eyereasoner.github.io/eyeling/vocab/message#>.\n` +
+      `@prefix log: <http://www.w3.org/2000/10/swap/log#>.\n` +
+      `{\n` +
+      `  ?Envelope eymsg:payloadGraph ?Payload.\n` +
+      `  ?Payload log:nameOf ?PayloadContext.\n` +
+      `  ?PayloadContext log:includes { ?Subject :line ?Line. }.\n` +
+      `} => {\n` +
+      `  ?Envelope log:outputString ?Line.\n` +
+      `}.\n`,
+    'utf8',
+  );
+}
+
+function writeBasicMessageLog(file) {
+  fs.writeFileSync(
+    file,
+    `VERSION "1.2-messages"\n` +
+      `PREFIX : <urn:test#>\n` +
+      `\n` +
+      `:a :line "one\\n".\n` +
+      `MESSAGE\n` +
+      `\n` +
+      `:b :line "two\\n".\n` +
+      `MESSAGE\n` +
+      `# empty heartbeat\n` +
+      `MESSAGE\n` +
+      `:c :line "three\\n".\n`,
+    'utf8',
+  );
+}
+
+function writeLargeMessageLog(file, count) {
+  let text = 'VERSION "1.2-messages"\nPREFIX : <urn:test#>\n';
+  for (let i = 1; i <= count; i += 1) {
+    text += `:m${i} :line "${i}\\n".\n`;
+    if (i < count) text += 'MESSAGE\n';
+  }
+  fs.writeFileSync(file, text, 'utf8');
+}
+
+function writeMarcMessageLog(file) {
+  fs.writeFileSync(
+    file,
+    `VERSION "1.2-messages"\n` +
+      `PREFIX : <http://example.org/ns#>\n` +
+      `\n` +
+      `:record1 :record (\n` +
+      `  ("001" "_" "_" "_" "42")\n` +
+      `  ("245" "1" "0" "a" "Streaming RDF Messages")\n` +
+      `  ("650" "_" "0" "a" "Semantic Web")\n` +
+      `  ("920" "_" "_" "a" "book")\n` +
+      `).\n` +
+      `MESSAGE\n` +
+      `:record2 :record (\n` +
+      `  ("001" "_" "_" "_" "43")\n` +
+      `  ("245" "1" "0" "a" "Incremental MARC Extraction")\n` +
+      `  ("650" "_" "0" "a" "Linked data")\n` +
+      `  ("920" "_" "_" "a" "article")\n` +
+      `).\n`,
+    'utf8',
+  );
+}
+
+const cases = [
+  {
+    name: 'scoped payload rules run once per RDF Message',
+    run(tmp) {
+      const rules = path.join(tmp, 'rules.n3');
+      const log = path.join(tmp, 'messages.trig');
+      writeScopedPayloadRules(rules);
+      writeBasicMessageLog(log);
+
+      const out = expectEyelingOk(['-r', '--stream-messages', rules, log]);
+      assert.equal(out, 'one\ntwo\nthree\n');
+    },
+  },
+  {
+    name: 'empty heartbeat messages do not leak previous payloads',
+    run(tmp) {
+      const rules = path.join(tmp, 'rules.n3');
+      const log = path.join(tmp, 'messages.trig');
+      writeScopedPayloadRules(rules);
+      writeBasicMessageLog(log);
+
+      const out = expectEyelingOk(['-r', '--stream-messages', rules, log]);
+      assert.equal(out.trim().split('\n').length, 3);
+      assert.ok(!out.includes('one\none'));
+    },
+  },
+  {
+    name: 'large message logs stream without requiring one monolithic dataset',
+    run(tmp) {
+      const rules = path.join(tmp, 'rules.n3');
+      const log = path.join(tmp, 'large.trig');
+      writeScopedPayloadRules(rules);
+      writeLargeMessageLog(log, 1000);
+
+      const out = expectEyelingOk(['-r', '--stream-messages', rules, log]);
+      const lines = out.trim().split('\n');
+      assert.equal(lines.length, 1000);
+      assert.equal(new Set(lines).size, 1000);
+      assert.ok(lines.includes('1'));
+      assert.ok(lines.includes('999'));
+      assert.ok(lines.includes('1000'));
+    },
+  },
+  {
+    name: 'MARC extraction rules fire over each streamed payload graph',
+    run(tmp) {
+      const log = path.join(tmp, 'marc.messages.txt');
+      writeMarcMessageLog(log);
+
+      const fixture = path.join(root, 'test', 'fixtures', 'marc-rules-stream-messages.n3');
+      const out = expectEyelingOk(['-r', '--stream-messages', fixture, log]);
+      assert.deepEqual(out.trim().split('\n').sort(), [
+        '<http://lib.ugent.be/record/42> <http://example.org/ns#subject> "Semantic Web" .',
+        '<http://lib.ugent.be/record/42> <http://example.org/ns#title> "Streaming RDF Messages" .',
+        '<http://lib.ugent.be/record/42> <http://example.org/ns#type> "book" .',
+        '<http://lib.ugent.be/record/43> <http://example.org/ns#subject> "Linked data" .',
+        '<http://lib.ugent.be/record/43> <http://example.org/ns#title> "Incremental MARC Extraction" .',
+        '<http://lib.ugent.be/record/43> <http://example.org/ns#type> "article" .',
+      ]);
+    },
+  },
+  {
+    name: '--stream-messages requires RDF mode',
+    run(tmp) {
+      const rules = path.join(tmp, 'rules.n3');
+      const log = path.join(tmp, 'messages.trig');
+      writeScopedPayloadRules(rules);
+      writeBasicMessageLog(log);
+
+      const r = runEyeling(['--stream-messages', rules, log]);
+      assert.notEqual(r.status, 0);
+      assert.match(r.stderr, /requires -r\/--rdf/);
+    },
+  },
+];
+
+(function main() {
+  const suiteStart = msNow();
+  info(`Running ${cases.length} stream-message tests`);
+
+  let passed = 0;
+  let failed = 0;
+
+  for (const [index, tc] of cases.entries()) {
+    const tmp = mkTmpDir();
+    const testName = numberedName(index, tc.name);
+    const start = msNow();
+
+    try {
+      tc.run(tmp);
+      ok(`${testName} ${C.dim}(${msNow() - start} ms)${C.n}`);
+      passed++;
+    } catch (e) {
+      fail(`${testName} ${C.dim}(${msNow() - start} ms)${C.n}`);
+      fail(e && e.stack ? e.stack : String(e));
+      failed++;
+    } finally {
+      rmrf(tmp);
+    }
+  }
+
+  console.log('');
+  const suiteMs = msNow() - suiteStart;
+  console.log(`${C.y}==${C.n} Total elapsed: ${suiteMs} ms (${(suiteMs / 1000).toFixed(2)} s)`);
+
+  if (failed === 0) {
+    ok(`All stream-message tests passed (${passed}/${cases.length})`);
+    process.exit(0);
+  }
+
+  fail(`Some stream-message tests failed (${passed}/${cases.length})`);
+  process.exit(1);
+})();