n3.js-messages 0.1.0

package/src/MessageScanner.js

@@ -0,0 +1,390 @@
+// **MessageScanner** finds RDF Message delimiters in a text stream.
+//
+// N3.js has no extension point for new directives, and its lexer rejects
+// `MESSAGE` and `@message`. The scanners in this file therefore sit in front of
+// the N3.js lexer: they track just enough lexical context (IRIs, strings,
+// long strings, comments, words, braces) to recognize *real* delimiters, and
+// rewrite them into text that N3.js can lex and that is easy to recognize
+// again in N3.js' public parser callbacks:
+//
+// - Turtle and TriG: a delimiter becomes `@version "<sentinel>" .`.
+//   N3.js only accepts version directives at statement boundaries, so the
+//   N3.js parser itself rejects delimiters in other grammar positions, and
+//   reports the delimiter through its public `onVersion` callback, exactly in
+//   order with the emitted quads. The dot matters: the N3.js lexer reads an
+//   `@` after a literal as a language tag, so a literal that ends a SPARQL-style
+//   `VERSION "label"` directive would swallow a following `@prefix`.
+//   For the same reason, SPARQL-style `VERSION "label"` directives in the input
+//   are rewritten to `@version "label" .`.
+// - N-Triples and N-Quads: N3.js has no version directive in line mode.
+//   The grammar is line-based, so a delimiter or version line becomes the
+//   comment `#<sentinel>M<line>` / `#<sentinel>V<label>`, which is reported through
+//   the public `onComment` callback, again in order with the quads.
+//
+// Delimiters that are not recognized (inside strings, IRIs, comments, prefixed
+// names or graph blocks) are passed through unchanged. The scanners never
+// throw: whatever they do not rewrite is rejected by N3.js as a syntax error.
+
+const TOP = 0, IRI = 1, LT = 2, COMMENT = 3, STRING = 4, QUOTE1 = 5, QUOTE2 = 6, LONG = 7, CANDIDATE = 8;
+
+const MESSAGE = 'MESSAGE', AT_MESSAGE = '@message', VERSION = 'VERSION';
+
+function isWhitespace(code) {
+  return code === 32 || code === 9 || code === 10 || code === 13;
+}
+
+// Compares a character to a position of a keyword; `VERSION` is case-insensitive
+function matches(keyword, index, code) {
+  return keyword === VERSION ? String.fromCharCode(code).toUpperCase() === keyword[index] :
+    code === keyword.charCodeAt(index);
+}
+
+// Characters that may follow `MESSAGE`: whitespace, a comment, or an IRI
+function endsMessageKeyword(code) {
+  return isWhitespace(code) || code === 35 || code === 60;
+}
+
+// Characters that may follow `@message`: whitespace, a comment, or the dot
+function endsAtMessageKeyword(code) {
+  return isWhitespace(code) || code === 35 || code === 46;
+}
+
+// ## Turtle and TriG
+export class TurtleScanner {
+  constructor(sentinel) {
+    this._sentinel = sentinel;
+    this._state = TOP;
+    // Whether the next character can start a new token
+    this._tokenStart = true;
+    // Whether we are inside a bare word (prefixed name, number, keyword)
+    this._inWord = false;
+    this._wordEscape = false;
+    this._quote = 0;
+    this._escaped = false;
+    this._quoteRun = 0;
+    this._depth = 0;
+    this._candidate = '';
+    this._target = '';
+    this._candidateLine = 1;
+    this._beforeCandidate = 32;
+    this._versionPending = false;
+    this._line = 1;
+    this._afterCR = false;
+    this._started = false;
+    this._lastOut = 32;
+    // The line of every rewritten delimiter that was not yet consumed
+    this.delimiterLines = [];
+  }
+
+  // ### `push` scans a chunk of text and returns the text to hand to N3.js
+  push(text) {
+    let out = '';
+    let segmentStart = 0;
+    for (let i = 0; i < text.length; i++) {
+      const code = text.charCodeAt(i);
+      // Keep track of the line number (CRLF counts once, also across chunks)
+      if (code === 10) {
+        if (!this._afterCR)
+          this._line++;
+        this._afterCR = false;
+      }
+      else if (code === 13) {
+        this._line++;
+        this._afterCR = true;
+      }
+      else
+        this._afterCR = false;
+
+      // A state change may require the same character to be handled again
+      let again;
+      do {
+        again = false;
+        switch (this._state) {
+        case TOP:
+          if (this._scanTop(code)) {
+            // A possible delimiter starts here: hold it back until it is resolved
+            out += text.substring(segmentStart, i);
+            segmentStart = i + 1;
+            this._candidate = text[i];
+            this._candidateLine = this._line;
+            this._beforeCandidate = i > 0 ? text.charCodeAt(i - 1) : this._lastOut;
+            this._target = code === 77 ? MESSAGE : code === 64 ? AT_MESSAGE : VERSION;
+            this._state = CANDIDATE;
+          }
+          break;
+        case CANDIDATE: {
+          const index = this._candidate.length, target = this._target;
+          if (index < target.length && matches(target, index, code)) {
+            this._candidate += text[i];
+            segmentStart = i + 1;
+          }
+          else {
+            const complete = index === target.length &&
+              (target === AT_MESSAGE ? endsAtMessageKeyword(code) : endsMessageKeyword(code));
+            out += this._resolveCandidate(complete);
+            segmentStart = i;
+            again = true;
+          }
+          break;
+        }
+        case IRI:
+          if (code === 62)
+            this._endToken(true);
+          break;
+        case LT:
+          // `<<` starts a reified triple, everything else an IRI
+          if (code === 60)
+            this._endToken(true);
+          else {
+            this._state = IRI;
+            again = true;
+          }
+          break;
+        case COMMENT:
+          if (code === 10 || code === 13)
+            this._endToken(true);
+          break;
+        case STRING:
+          if (this._escaped)
+            this._escaped = false;
+          else if (code === 92)
+            this._escaped = true;
+          else if (code === this._quote) {
+            this._endToken(false);
+            if (this._versionPending) {
+              // Terminate a SPARQL-style version directive, which we turned into an `@version` directive
+              this._versionPending = false;
+              out += `${text.substring(segmentStart, i + 1)} .`;
+              segmentStart = i + 1;
+            }
+          }
+          break;
+        case QUOTE1:
+          // A second quote either starts a long string or ends an empty string
+          if (code === this._quote)
+            this._state = QUOTE2;
+          else {
+            this._state = STRING;
+            again = true;
+          }
+          break;
+        case QUOTE2:
+          if (code === this._quote) {
+            this._state = LONG;
+            this._escaped = false;
+            this._quoteRun = 0;
+          }
+          else {
+            this._endToken(false);
+            again = true;
+          }
+          break;
+        default: // LONG
+          if (this._escaped) {
+            this._escaped = false;
+            this._quoteRun = 0;
+          }
+          else if (code === 92) {
+            this._escaped = true;
+            this._quoteRun = 0;
+          }
+          else if (code === this._quote) {
+            if (++this._quoteRun === 3)
+              this._endToken(false);
+          }
+          else
+            this._quoteRun = 0;
+        }
+      } while (again);
+    }
+    out += text.substring(segmentStart);
+    return this._remember(out);
+  }
+
+  // ### `end` resolves any held-back text at the end of the input
+  end() {
+    let out = '';
+    if (this._state === CANDIDATE)
+      out = this._resolveCandidate(this._candidate.length === this._target.length);
+    return this._remember(out);
+  }
+
+  // ### `_scanTop` handles a character between tokens or inside a word.
+  // It returns true if the character may start a message delimiter.
+  _scanTop(code) {
+    // An escape inside a word makes the next character part of the word
+    if (this._wordEscape) {
+      this._wordEscape = false;
+      return false;
+    }
+    switch (code) {
+    case 32: case 9: case 10: case 13:
+      this._structural();
+      break;
+    case 0xFEFF:
+      // A byte order mark at the very start is not part of any token
+      if (this._started)
+        this._word();
+      break;
+    case 35: // #
+      this._state = COMMENT;
+      this._inWord = false;
+      break;
+    case 60: // <
+      this._state = LT;
+      this._inWord = false;
+      break;
+    case 34: case 39: // " '
+      this._state = QUOTE1;
+      this._quote = code;
+      this._inWord = false;
+      break;
+    case 123: // {
+      this._depth++;
+      this._structural();
+      break;
+    case 125: // }
+      this._depth = Math.max(0, this._depth - 1);
+      this._structural();
+      break;
+    case 40: case 41: case 91: case 93: case 44: case 59: // ( ) [ ] , ;
+      this._structural();
+      break;
+    case 46: // . ends a statement, unless it is part of a word
+      if (!this._inWord)
+        this._structural();
+      break;
+    case 92: // \
+      this._word();
+      this._wordEscape = true;
+      break;
+    case 77: case 64: case 86: case 118: // M @ V v
+      if (this._tokenStart) {
+        this._started = true;
+        return true;
+      }
+      this._word();
+      break;
+    default:
+      this._word();
+    }
+    this._started = true;
+    return false;
+  }
+
+  _structural() {
+    this._inWord = false;
+    this._tokenStart = true;
+  }
+
+  _word() {
+    this._inWord = true;
+    this._tokenStart = false;
+  }
+
+  // ### `_endToken` returns to the top level after an IRI, string or comment
+  _endToken(tokenStart) {
+    this._state = TOP;
+    this._inWord = false;
+    this._tokenStart = tokenStart;
+  }
+
+  // ### `_resolveCandidate` decides whether a held-back candidate is a delimiter
+  // and returns the text to emit for it
+  _resolveCandidate(complete) {
+    const candidate = this._candidate;
+    this._state = TOP;
+    this._candidate = '';
+    // A delimiter in a graph block is not rewritten, so that N3.js rejects it
+    if (!complete || (this._depth > 0 && this._target !== VERSION)) {
+      this._word();
+      return candidate;
+    }
+    this._structural();
+    const previous = this._beforeCandidate;
+    const space = isWhitespace(previous) || previous === 0xFEFF ? '' : ' ';
+    if (this._target === VERSION) {
+      this._versionPending = true;
+      return `${space}@version`;
+    }
+    this.delimiterLines.push(this._candidateLine);
+    return `${space}@version "${this._sentinel}"${candidate === MESSAGE ? ' .' : ''}`;
+  }
+
+  _remember(out) {
+    if (out.length > 0)
+      this._lastOut = out.charCodeAt(out.length - 1);
+    return out;
+  }
+}
+
+// ## N-Triples and N-Quads
+// Lines of the form `MESSAGE` or `VERSION "label"` are the only directives.
+// They cannot be confused with strings or IRIs, as those never span lines.
+export class LineScanner {
+  constructor(sentinel) {
+    this._sentinel = sentinel;
+    this._buffer = '';
+    this._line = 1;
+    this._messageLine = new RegExp(`^[ \\t]*${MESSAGE}[ \\t]*(?:#.*)?$`);
+    this._versionLine = /^[ \t]*VERSION[ \t]*(?:"([^"\\]*)"|'([^'\\]*)')[ \t]*(?:#.*)?$/i;
+  }
+
+  // ### `push` scans a chunk of text and returns the complete lines, rewritten
+  push(text) {
+    const data = this._buffer + text;
+    // A final carriage return waits for a possible line feed, so that CRLF counts as one line break
+    const limit = data.endsWith('\r') ? data.length - 1 : data.length;
+    const lastBreak = Math.max(data.lastIndexOf('\n', limit - 1), data.lastIndexOf('\r', limit - 1));
+    if (lastBreak < 0) {
+      this._buffer = data;
+      return '';
+    }
+    this._buffer = data.substring(lastBreak + 1);
+    return this._rewrite(data.substring(0, lastBreak + 1));
+  }
+
+  // ### `end` returns the last line, which has no line break
+  end() {
+    const rest = this._buffer;
+    this._buffer = '';
+    return rest.length > 0 ? this._rewrite(rest) : '';
+  }
+
+  _rewrite(block) {
+    let out = '';
+    let copyFrom = 0, lineStart = 0;
+    const breaks = /\r\n|\n|\r/g;
+    let match;
+    do {
+      match = breaks.exec(block);
+      const lineEnd = match ? match.index : block.length;
+      const replacement = this._rewriteLine(block, lineStart, lineEnd);
+      if (replacement !== null) {
+        out += block.substring(copyFrom, lineStart) + replacement;
+        copyFrom = lineEnd;
+      }
+      lineStart = match ? breaks.lastIndex : block.length + 1;
+      if (match)
+        this._line++;
+    } while (lineStart <= block.length);
+    return out + block.substring(copyFrom);
+  }
+
+  // Only lines that start with `MESSAGE` or `VERSION` need a closer look
+  _rewriteLine(block, start, end) {
+    let first = start;
+    while (first < end && (block[first] === ' ' || block[first] === '\t'))
+      first++;
+    const code = block.charCodeAt(first);
+    if (first === end || (code !== 77 && code !== 86 && code !== 118))
+      return null;
+    const line = block.substring(first, end);
+    if (this._messageLine.test(line))
+      return `#${this._sentinel}M${this._line}`;
+    const version = this._versionLine.exec(line);
+    if (version)
+      return `#${this._sentinel}V${version[1] === undefined ? version[2] : version[1]}`;
+    return null;
+  }
+}

package/src/MessageStreamParser.js

@@ -0,0 +1,33 @@
+// **MessageStreamParser** is a Node.js stream that turns text into RDF Messages.
+//
+// It consumes strings or buffers, and produces one array of quads per message,
+// as soon as that message is complete.
+import { Transform } from 'stream';
+import { StringDecoder } from 'string_decoder';
+import MessageParserCore from './MessageParserCore.js';
+
+export default class MessageStreamParser extends Transform {
+  constructor(options) {
+    super({ decodeStrings: false, readableObjectMode: true });
+    this._decoder = new StringDecoder('utf8');
+    this._error = null;
+    this._core = new MessageParserCore(options, {
+      message: quads => this.push(quads),
+      end: () => {},
+      error: error => { this._error = this._error || error; },
+      prefix: (prefix, term) => this.emit('prefix', prefix, term),
+      version: label => this.emit('version', label),
+    });
+  }
+
+  _transform(chunk, encoding, done) {
+    this._core.write(typeof chunk === 'string' ? chunk : this._decoder.write(chunk));
+    done(this._error);
+  }
+
+  _flush(done) {
+    this._core.write(this._decoder.end());
+    this._core.end();
+    done(this._error);
+  }
+}

package/src/MessageStreamWriter.js

@@ -0,0 +1,22 @@
+// **MessageStreamWriter** is a Node.js stream that serializes RDF Messages.
+//
+// It consumes messages (arrays of quads) and produces text.
+import { Transform } from 'stream';
+import MessageWriter from './MessageWriter.js';
+
+export default class MessageStreamWriter extends Transform {
+  constructor(options) {
+    super({ writableObjectMode: true, encoding: 'utf8' });
+    // Writing into this stream never blocks, and output is pushed synchronously
+    const output = { write: (chunk, encoding, done) => (this.push(chunk), done && done()) };
+    this._writer = new MessageWriter(output, Object.assign({}, options, { end: false }));
+  }
+
+  _transform(message, encoding, done) {
+    this._writer.addMessage(message, done);
+  }
+
+  _flush(done) {
+    this._writer.end(done);
+  }
+}

package/src/MessageWriter.js

@@ -0,0 +1,194 @@
+// **MessageWriter** writes RDF Message Logs (Turtle, TriG, N-Triples and N-Quads).
+//
+// Every message is written by a fresh N3.js writer, so no serialization state
+// (current graph, subject, predicate) leaks from one message into the next.
+// Every message is followed by a delimiter (`MESSAGE` in line formats, `@message .` in Turtle and TriG), so that consumers receive
+// a message as soon as it is written, and so that trailing empty messages survive
+// a round trip (a delimiter before the end of the input does not create a message).
+import { Writer } from 'n3';
+import { resolveFormat, isMessagesVersion, baseVersion, isSupportedVersion } from './formats.js';
+
+export default class MessageWriter {
+  // `outputStream` is optional: without it, `end` returns the output as a string
+  constructor(outputStream, options) {
+    if (outputStream && typeof outputStream.write !== 'function')
+      options = outputStream, outputStream = null;
+    options = options || {};
+    this._format = resolveFormat(options.format);
+    this._options = options;
+    this._version = options.version === undefined ? '1.2-messages' : options.version;
+    if (!isMessagesVersion(this._version) || !isSupportedVersion(baseVersion(this._version)))
+      throw new TypeError(`Unsupported RDF Messages version: "${this._version}"`);
+
+    this._output = '';
+    this._stream = outputStream;
+    this._endStream = !outputStream || options.end !== false;
+    this._pending = 0;
+    this._waiting = [];
+    this._error = null;
+    this._muted = false;
+    this._started = false;
+    this._ended = false;
+    this._declared = Object.assign({}, options.prefixes);
+    this._newPrefixes = {};
+    // The N3.js writers write to this sink, which they never end
+    this._sink = { write: (chunk, encoding, done) => this._write(chunk, done) };
+  }
+
+  // ### `addMessage` writes the quads of one message, followed by a delimiter.
+  // The callback (or returned promise) completes once the output is flushed.
+  addMessage(quads, done) {
+    return this._complete(done, () => {
+      this._start();
+      this._writeMessage(quads || []);
+    });
+  }
+
+  // ### `addPrefix` declares a prefix, which is written before the next message
+  // and stays in effect for all messages after it
+  addPrefix(prefix, iri, done) {
+    return this._complete(done, () => {
+      if (!this._format.lineMode && this._declared[prefix] !== (iri.value || iri))
+        this._newPrefixes[prefix] = iri.value || iri;
+    });
+  }
+
+  // ### `end` ends the log and, unless disabled, the output stream.
+  // The callback receives the output as a string if no output stream was given.
+  end(done) {
+    // Ending twice is harmless
+    if (this._ended)
+      return typeof done === 'function' ? done(null) : Promise.resolve();
+    this._start();
+    if (typeof done === 'function')
+      return this._finish(done);
+    return new Promise((resolve, reject) => this._finish((error, result) => error ? reject(error) : resolve(result)));
+  }
+
+  // ## Private methods
+
+  // Runs an action and reports completion with a callback or a promise
+  _complete(done, action) {
+    if (this._ended)
+      this._error = this._error || new Error('Cannot write after end');
+    else
+      try {
+        action();
+      }
+      catch (error) {
+        this._error = this._error || error;
+      }
+    if (typeof done === 'function')
+      return this._whenFlushed(done);
+    return new Promise((resolve, reject) => this._whenFlushed(error => error ? reject(error) : resolve()));
+  }
+
+  // Ends the output once all queued writes were accepted
+  _finish(callback) {
+    this._ended = true;
+    this._whenFlushed(error => this._endOutput(error, callback));
+  }
+
+  _endOutput(error, callback) {
+    if (error || !this._endStream)
+      return callback(error);
+    if (!this._stream)
+      return callback(null, this._output);
+    try {
+      this._stream.end(endError => callback(endError));
+    }
+    catch (endError) {
+      callback(endError);
+    }
+  }
+
+  // Calls back once all queued writes were accepted by the output
+  _whenFlushed(done) {
+    if (this._pending === 0)
+      this._settle(done);
+    else
+      this._waiting.push(done);
+  }
+
+  _settle(done) {
+    const error = this._error;
+    this._error = null;
+    done(error || null);
+  }
+
+  _write(chunk, done) {
+    if (this._muted || !this._stream) {
+      if (!this._muted)
+        this._output += chunk;
+      return done && done();
+    }
+    this._pending++;
+    try {
+      this._stream.write(chunk, 'utf8', error => this._written(error, done));
+    }
+    catch (error) {
+      this._written(error, done);
+    }
+  }
+
+  // Registers that the output accepted a write, and wakes up those waiting for the output
+  _written(error, done) {
+    this._pending--;
+    this._error = this._error || error || null;
+    done && done(error);
+    if (this._pending === 0)
+      this._waiting.splice(0).forEach(callback => this._settle(callback));
+  }
+
+  // Announces the RDF Messages version, which makes parsers expect delimiters
+  _start() {
+    if (!this._started) {
+      this._started = true;
+      this._write(this._format.lineMode ? `VERSION "${this._version}"\n` : `@version "${this._version}" .\n`);
+    }
+  }
+
+  _writeMessage(quads) {
+    const first = !this._messageWritten;
+    const options = { format: this._format.name, end: false };
+    if (this._options.baseIRI) {
+      options.baseIRI = this._options.baseIRI;
+      // The base is declared by us, once, as it stays in effect (and older N3.js writers lack `writeBase`)
+      if (first && this._options.writeBase && !this._format.lineMode)
+        this._write(`@base <${this._options.baseIRI}>.\n`);
+    }
+    // Prefixes and base are declared once, as they stay in effect
+    options.prefixes = this._declared;
+    this._muted = !first;
+    const writer = new Writer(this._sink, options);
+    this._muted = false;
+    this._messageWritten = true;
+    for (const prefix of Object.keys(this._newPrefixes)) {
+      writer.addPrefix(prefix, this._newPrefixes[prefix]);
+      this._declared[prefix] = this._newPrefixes[prefix];
+    }
+    this._newPrefixes = {};
+    const record = error => error && (this._error = this._error || error);
+    for (const quad of quads)
+      writer.addQuad(quad.subject, quad.predicate, quad.object, quad.graph, record);
+    writer.end(record);
+    this._write(this._format.lineMode ? 'MESSAGE\n' : '@message .\n');
+  }
+}
+
+// ### `writeMessages` synchronously serializes an array of messages to a string
+export function writeMessages(messages, options) {
+  const writer = new MessageWriter(options);
+  const errors = [];
+  let output;
+  function collect(error) {
+    if (error)
+      errors.push(error);
+  }
+  for (const message of messages)
+    writer.addMessage(message, collect);
+  writer.end((error, result) => (collect(error), output = result));
+  if (errors.length > 0)
+    throw errors[0];
+  return output;
+}

package/src/formats.js

@@ -0,0 +1,39 @@
+// **formats** describes the RDF syntaxes that support RDF Messages.
+
+const FORMATS = {
+  turtle: { name: 'Turtle', lineMode: false },
+  trig: { name: 'TriG', lineMode: false },
+  ntriples: { name: 'N-Triples', lineMode: true },
+  nquads: { name: 'N-Quads', lineMode: true },
+};
+
+// The version labels of RDF 1.2 syntaxes, as accepted by N3.js
+const BASE_VERSIONS = ['1.2', '1.2-basic', '1.1'];
+
+// ### `resolveFormat` maps a format name or media type to a supported syntax
+export function resolveFormat(format = 'TriG') {
+  const key = String(format).toLowerCase().split(';')[0].trim()
+    .replace(/^(?:text|application)\//, '')
+    .replace(/^x-/, '')
+    .replace(/[-_ ]/g, '');
+  const resolved = FORMATS[key];
+  if (!resolved)
+    throw new TypeError(`Unsupported RDF Messages format: "${format}". ` +
+                        'Supported formats are Turtle, TriG, N-Triples and N-Quads.');
+  return resolved;
+}
+
+// ### `isMessagesVersion` checks whether a version label announces RDF Messages
+export function isMessagesVersion(version) {
+  return typeof version === 'string' && version.endsWith('-messages');
+}
+
+// ### `baseVersion` strips the `-messages` suffix of a version label
+export function baseVersion(version) {
+  return isMessagesVersion(version) ? version.slice(0, -'-messages'.length) : version;
+}
+
+// ### `isSupportedVersion` checks whether a (base) version label is supported
+export function isSupportedVersion(version, supported = BASE_VERSIONS) {
+  return supported.includes(version);
+}

package/src/index.js

@@ -0,0 +1,4 @@
+export { default as MessageParser, parseMessages } from './MessageParser.js';
+export { default as MessageStreamParser } from './MessageStreamParser.js';
+export { default as MessageWriter, writeMessages } from './MessageWriter.js';
+export { default as MessageStreamWriter } from './MessageStreamWriter.js';