mdq-cli 0.1.0

package/index.js

@@ -0,0 +1,635 @@
+// src/utils/mdq/query.ts
+import { marked } from "marked";
+
+// src/utils/mdq/edit.ts
+import YAML from "yaml";
+
+class MarkdownEditor {
+  source;
+  constructor(source) {
+    this.source = source;
+  }
+  static splitFrontmatter(source) {
+    if (!source.startsWith("---"))
+      return { raw: "", body: source, offset: 0 };
+    const match = source.match(/^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/);
+    if (!match)
+      return { raw: "", body: source, offset: 0 };
+    return { raw: match[1], body: source.slice(match[0].length), offset: match[0].length };
+  }
+  static blockEnd(range) {
+    if (range.trailing)
+      return range.trailing.start + range.trailing.length;
+    return range.start + range.length;
+  }
+  static table(headers, rows, align) {
+    const widths = headers.map((header, index) => Math.max(header.length, 3, ...rows.map((row) => (row[index] || "").length)));
+    const line = (cells) => `| ${cells.map((cell, index) => (cell || "").padEnd(widths[index])).join(" | ")} |`;
+    const divider = `| ${widths.map((width, index) => MarkdownEditor.dashes(align[index], width)).join(" | ")} |`;
+    return `${[line(headers), divider, ...rows.map(line)].join(`
+`)}
+`;
+  }
+  static item(listRaw, text) {
+    const lines = listRaw.split(`
+`).filter((line) => line.trim());
+    const last = lines[lines.length - 1] || "- x";
+    const ordered = last.match(/^(\s*)(\d+)([.)])\s/);
+    if (ordered)
+      return `${ordered[1]}${Number.parseInt(ordered[2], 10) + 1}${ordered[3]} ${text}`;
+    const bullet = last.match(/^(\s*)([-*+])\s/);
+    if (!bullet)
+      return `- ${text}`;
+    return `${bullet[1]}${bullet[2]} ${text}`;
+  }
+  static entries(text, key, value, isBlockquote) {
+    const lines = text.split(`
+`).map((line) => line.trim()).filter(Boolean);
+    const index = lines.findIndex((line) => MarkdownEditor.entryKey(line) === key.toLowerCase());
+    if (index < 0 && value)
+      lines.push(`${key}: ${value}`);
+    if (index >= 0 && value)
+      lines[index] = `${key}: ${value}`;
+    if (index >= 0 && !value)
+      lines.splice(index, 1);
+    if (!isBlockquote)
+      return lines.join(`
+`);
+    return lines.map((line) => `> ${line}`).join(`
+`);
+  }
+  remove(ranges) {
+    const ordered = this.dedupe(ranges);
+    let result = this.source;
+    for (let i = ordered.length - 1;i >= 0; i--) {
+      const range = ordered[i];
+      const head = result.slice(0, range.start);
+      const tail = result.slice(MarkdownEditor.blockEnd(range));
+      if (tail) {
+        result = head + tail;
+        continue;
+      }
+      if (!head) {
+        result = "";
+        continue;
+      }
+      result = `${head.trimEnd()}
+`;
+    }
+    return result;
+  }
+  replace(ranges, render) {
+    const ordered = this.dedupe(ranges);
+    const rendered = ordered.map(render);
+    let result = this.source;
+    for (let i = ordered.length - 1;i >= 0; i--) {
+      const range = ordered[i];
+      result = result.slice(0, range.start) + rendered[i] + result.slice(range.start + range.length);
+    }
+    return result;
+  }
+  insert(offsets, markdown) {
+    const ordered = [...offsets].sort((a, b) => a - b);
+    let result = this.source;
+    for (let i = ordered.length - 1;i >= 0; i--) {
+      result = MarkdownEditor.spliceIn(result, ordered[i], markdown);
+    }
+    return result;
+  }
+  frontmatter() {
+    const { raw } = MarkdownEditor.splitFrontmatter(this.source);
+    if (!raw)
+      return {};
+    return YAML.parseDocument(raw).toJS() || {};
+  }
+  setFrontmatter(key, value) {
+    const { raw, body, offset } = MarkdownEditor.splitFrontmatter(this.source);
+    let document = new YAML.Document({});
+    if (raw)
+      document = YAML.parseDocument(raw);
+    if (value === null)
+      document.delete(key);
+    if (value !== null)
+      document.set(key, value);
+    const rendered = document.toString().trimEnd();
+    if (!offset)
+      return `---
+${rendered}
+---
+
+${this.source}`;
+    return `---
+${rendered}
+---
+${body}`;
+  }
+  static spliceIn(source, offset, markdown) {
+    const block = `${markdown.trimEnd()}
+`;
+    const before = source.slice(0, offset).replace(/\n+$/, "");
+    const after = source.slice(offset).replace(/^\n+/, "");
+    if (!before)
+      return `${block}
+${after}`;
+    if (!after)
+      return `${before}
+
+${block}`;
+    return `${before}
+
+${block}
+${after}`;
+  }
+  static dashes(alignment, width) {
+    if (alignment === "center")
+      return `:${"-".repeat(Math.max(width - 2, 1))}:`;
+    if (alignment === "left")
+      return `:${"-".repeat(Math.max(width - 1, 1))}`;
+    if (alignment === "right")
+      return `${"-".repeat(Math.max(width - 1, 1))}:`;
+    return "-".repeat(width);
+  }
+  static entryKey(line) {
+    const separator = line.indexOf(":");
+    if (separator < 1)
+      return null;
+    return line.slice(0, separator).trim().toLowerCase();
+  }
+  dedupe(ranges) {
+    const sorted = [...ranges].sort((a, b) => a.start - b.start);
+    const kept = [];
+    let lastEnd = -1;
+    for (const range of sorted) {
+      if (range.start < lastEnd)
+        continue;
+      kept.push(range);
+      lastEnd = range.start + range.length;
+    }
+    return kept;
+  }
+}
+
+// src/utils/mdq/query.ts
+class MarkdownQuery {
+  source;
+  matches;
+  constructor(source, matches) {
+    this.source = source;
+    this.matches = matches || buildTokenIndex(source);
+  }
+  query(selector, matcher) {
+    const segments = parseQuery(selector);
+    if (matcher !== undefined && segments.length > 0)
+      segments[segments.length - 1].textMatch = this.valueMatcher(matcher);
+    return new MarkdownQuery(this.source, this.run(this.expandSections(this.matches), segments));
+  }
+  section(matcher, options) {
+    return this.query(`section${options?.depth || ""}`, matcher);
+  }
+  heading(matcher, options) {
+    if (options?.depth)
+      return this.query(`h${options.depth}`, matcher);
+    return this.query("heading", matcher);
+  }
+  paragraph(matcher) {
+    return this.query("paragraph", matcher);
+  }
+  table(matcher) {
+    return this.query("table", matcher);
+  }
+  list(matcher) {
+    return this.query("list", matcher);
+  }
+  item(matcher) {
+    return this.query("item", matcher);
+  }
+  code(matcher) {
+    return this.query("code", matcher);
+  }
+  blockquote(matcher) {
+    return this.query("blockquote", matcher);
+  }
+  comment(matcher) {
+    return this.query("comment", matcher);
+  }
+  html(matcher) {
+    return this.query("html", matcher);
+  }
+  hr() {
+    return this.query("hr");
+  }
+  text() {
+    return this.matches.map((range) => this.source.slice(range.start, range.start + range.length)).join("");
+  }
+  nodes() {
+    return this.matches.map((range) => {
+      const token = range.token;
+      if (token.type !== "heading")
+        return { type: token.type, depth: null, text: this.tokenText(range.token) };
+      return { type: token.type, depth: token.depth, text: this.tokenText(range.token) };
+    });
+  }
+  rows() {
+    const results = [];
+    for (const range of this.matches) {
+      if (range.token.type !== "table")
+        continue;
+      const table = range.token;
+      const headers = table.header.map((cell) => cell.text);
+      for (const row of table.rows) {
+        const entry = {};
+        for (let i = 0;i < headers.length; i++) {
+          entry[headers[i]] = row[i]?.text ?? "";
+        }
+        results.push(entry);
+      }
+    }
+    return results;
+  }
+  entries() {
+    const entries = {};
+    for (const range of this.matches) {
+      for (const line of this.tokenText(range.token).split(`
+`)) {
+        const separator = line.indexOf(":");
+        if (separator < 1)
+          continue;
+        const value = line.slice(separator + 1).trim();
+        if (value)
+          entries[line.slice(0, separator).trim().toLowerCase()] = value;
+      }
+    }
+    return entries;
+  }
+  frontmatter() {
+    return this.editor().frontmatter();
+  }
+  count() {
+    return this.matches.length;
+  }
+  exists() {
+    return this.matches.length > 0;
+  }
+  at(index) {
+    let resolved = index;
+    if (resolved < 0)
+      resolved = this.matches.length + resolved;
+    if (resolved < 0 || resolved >= this.matches.length)
+      return new MarkdownQuery(this.source, []);
+    return new MarkdownQuery(this.source, [this.matches[resolved]]);
+  }
+  slice(from, to) {
+    return new MarkdownQuery(this.source, this.matches.slice(from, to));
+  }
+  first() {
+    return this.slice(0, 1);
+  }
+  last() {
+    return this.slice(-1);
+  }
+  each() {
+    return this.matches.map((range) => new MarkdownQuery(this.source, [range]));
+  }
+  preceding() {
+    if (this.matches.length === 0)
+      return new MarkdownQuery(this.source, []);
+    const cutoff = this.matches[0].start;
+    return new MarkdownQuery(this.source, buildTokenIndex(this.source).filter((range) => range.start + range.length <= cutoff));
+  }
+  following() {
+    if (this.matches.length === 0)
+      return new MarkdownQuery(this.source, []);
+    const last = this.matches[this.matches.length - 1];
+    const cutoff = last.start + last.length;
+    return new MarkdownQuery(this.source, buildTokenIndex(this.source).filter((range) => range.start >= cutoff));
+  }
+  replace(content) {
+    return this.replaceEach(() => content);
+  }
+  replaceEach(replacer) {
+    return new MarkdownQuery(this.editor().replace(this.matches, (range, index) => String(replacer(new MarkdownQuery(this.source, [range]), index))));
+  }
+  remove() {
+    return new MarkdownQuery(this.editor().remove(this.matches));
+  }
+  insertBefore(markdown) {
+    return this.insertAll((range) => range.start, markdown);
+  }
+  insertAfter(markdown) {
+    return this.insertAll((range) => MarkdownEditor.blockEnd(range), markdown);
+  }
+  prepend(markdown) {
+    return this.insertAll((range) => this.containerStart(range), markdown);
+  }
+  append(markdown) {
+    return this.insertAll((range) => this.containerEnd(range), markdown);
+  }
+  addRow(row) {
+    return new MarkdownQuery(this.editor().replace(this.matches, (range) => {
+      if (range.token.type !== "table")
+        throw new MdqOperationError(`addRow needs a table, got ${range.token.type}`);
+      const table = range.token;
+      const headers = table.header.map((cell) => cell.text);
+      const existing = table.rows.map((cells) => headers.map((_, index) => cells[index]?.text || ""));
+      return MarkdownEditor.table(headers, [...existing, headers.map((header) => row[header] || "")], table.align);
+    }));
+  }
+  addItem(text) {
+    return new MarkdownQuery(this.editor().replace(this.matches, (range) => {
+      if (range.token.type !== "list")
+        throw new MdqOperationError(`addItem needs a list, got ${range.token.type}`);
+      const raw = (range.token.raw || "").trimEnd();
+      return `${raw}
+${MarkdownEditor.item(raw, text)}
+`;
+    }));
+  }
+  setEntry(key, value) {
+    return this.replaceEach((match) => {
+      const token = match.matches[0].token;
+      return MarkdownEditor.entries(this.tokenText(token), key, value, token.type === "blockquote");
+    });
+  }
+  setFrontmatter(key, value) {
+    return new MarkdownQuery(this.editor().setFrontmatter(key, value));
+  }
+  toString() {
+    return this.source;
+  }
+  valueOf() {
+    return this.source;
+  }
+  get() {
+    return this.text();
+  }
+  meta() {
+    return this.nodes();
+  }
+  toJson() {
+    return this.rows();
+  }
+  keyValue() {
+    return this.entries();
+  }
+  setKeyValue(key, value) {
+    return this.setEntry(key, value);
+  }
+  before() {
+    return this.preceding();
+  }
+  after() {
+    return this.following();
+  }
+  editor() {
+    return new MarkdownEditor(this.source);
+  }
+  run(candidates, segments) {
+    if (segments.length === 0)
+      return candidates;
+    const segment = segments[0];
+    const remaining = segments.slice(1);
+    if (/^section[1-6]?$/.test(segment.selector)) {
+      const sections = this.narrow(this.sectionsOf(candidates, segment), segment);
+      if (remaining.length === 0)
+        return sections;
+      const results = [];
+      for (const section of sections)
+        results.push(...this.run(section.innerTokens || [], remaining));
+      return results;
+    }
+    if (segment.selector === "comment") {
+      const comments = candidates.filter((range) => this.isComment(range.token));
+      return this.run(this.narrow(this.filterByText(comments, segment), segment), remaining);
+    }
+    if (segment.selector === "item") {
+      return this.run(this.narrow(this.filterByText(this.itemsOf(candidates), segment), segment), remaining);
+    }
+    const depth = segment.selector.match(/^h([1-6])$/);
+    let type = TOKEN_ALIASES[segment.selector] || segment.selector;
+    if (depth)
+      type = "heading";
+    let matches = candidates.filter((range) => range.token.type === type);
+    if (depth)
+      matches = matches.filter((range) => range.token.depth === Number.parseInt(depth[1], 10));
+    return this.run(this.narrow(this.filterByText(matches, segment), segment), remaining);
+  }
+  sectionsOf(candidates, segment) {
+    const wanted = segment.selector.match(/^section([1-6])$/);
+    const sections = [];
+    for (let i = 0;i < candidates.length; i++) {
+      const range = candidates[i];
+      if (range.token.type !== "heading")
+        continue;
+      const heading = range.token;
+      if (wanted && heading.depth !== Number.parseInt(wanted[1], 10))
+        continue;
+      if (segment.textMatch && !this.matchText(heading.text, segment.textMatch))
+        continue;
+      const innerTokens = [];
+      let end = range.start + range.length;
+      for (let j = i + 1;j < candidates.length; j++) {
+        const next = candidates[j];
+        if (next.token.type === "heading" && next.token.depth <= heading.depth)
+          break;
+        innerTokens.push(next);
+        end = MarkdownEditor.blockEnd(next);
+      }
+      sections.push({ token: range.token, start: range.start, length: end - range.start, innerTokens });
+    }
+    return sections;
+  }
+  itemsOf(candidates) {
+    const items = [];
+    for (const range of candidates) {
+      if (range.token.type !== "list")
+        continue;
+      const raw = range.token.raw;
+      let cursor = 0;
+      for (const item of range.token.items) {
+        const itemRaw = item.raw;
+        const at = raw.indexOf(itemRaw, cursor);
+        if (at === -1)
+          continue;
+        items.push({ token: item, start: range.start + at, length: itemRaw.length });
+        cursor = at + itemRaw.length;
+      }
+    }
+    return items;
+  }
+  expandSections(matches) {
+    if (!matches.some((range) => range.innerTokens))
+      return matches;
+    const expanded = [];
+    for (const range of matches) {
+      if (!range.innerTokens) {
+        expanded.push(range);
+        continue;
+      }
+      expanded.push({ token: range.token, start: range.start, length: (range.token.raw || "").length });
+      expanded.push(...range.innerTokens);
+    }
+    return expanded;
+  }
+  narrow(matches, segment) {
+    if (segment.index !== null) {
+      let index = segment.index;
+      if (index < 0)
+        index = matches.length + index;
+      if (index < 0 || index >= matches.length)
+        return [];
+      return [matches[index]];
+    }
+    if (segment.slice)
+      return matches.slice(segment.slice.from, segment.slice.to);
+    return matches;
+  }
+  filterByText(matches, segment) {
+    if (!segment.textMatch)
+      return matches;
+    return matches.filter((range) => this.matchText(this.tokenText(range.token), segment.textMatch));
+  }
+  matchText(text, matcher) {
+    let result = false;
+    if (matcher.mode === "exact")
+      result = text === matcher.value;
+    if (matcher.mode === "contains")
+      result = text.includes(matcher.value);
+    if (matcher.mode === "regex")
+      result = new RegExp(matcher.value, matcher.flags || "").test(text);
+    if (matcher.mode === "predicate")
+      result = matcher.predicate(text);
+    if (matcher.negated)
+      return !result;
+    return result;
+  }
+  valueMatcher(matcher) {
+    if (typeof matcher === "function")
+      return { mode: "predicate", value: "", negated: false, predicate: matcher };
+    if (matcher instanceof RegExp)
+      return { mode: "regex", value: matcher.source, negated: false, flags: matcher.flags };
+    return { mode: "exact", value: matcher, negated: false };
+  }
+  tokenText(token) {
+    const value = token;
+    if (this.isComment(token))
+      return value.raw.trim().replace(/^<!--/, "").replace(/-->$/, "").trim();
+    if (token.type === "html")
+      return value.raw || "";
+    if (token.type === "table")
+      return [...(value.header || []).map((cell) => cell.text), ...(value.rows || []).flatMap((row) => row.map((cell) => cell.text))].join(", ");
+    return value.text || "";
+  }
+  isComment(token) {
+    if (token.type !== "html")
+      return false;
+    return (token.raw || "").trimStart().startsWith("<!--");
+  }
+  insertAll(offsetOf, markdown) {
+    return new MarkdownQuery(this.editor().insert(this.matches.map(offsetOf), String(markdown)));
+  }
+  containerStart(range) {
+    if (!range.innerTokens)
+      throw new MdqOperationError(`prepend needs a section or list, got ${range.token.type}`);
+    return range.start + (range.token.raw || "").length;
+  }
+  containerEnd(range) {
+    if (!range.innerTokens)
+      throw new MdqOperationError(`append needs a section or list, got ${range.token.type}`);
+    const last = range.innerTokens[range.innerTokens.length - 1];
+    if (!last)
+      return this.containerStart(range);
+    return MarkdownEditor.blockEnd(last);
+  }
+}
+
+class MdqError extends Error {
+}
+
+class MdqSelectorError extends MdqError {
+  index;
+  constructor(message, index) {
+    super(message);
+    this.name = "MdqSelectorError";
+    this.index = index;
+  }
+}
+
+class MdqOperationError extends MdqError {
+  constructor(message) {
+    super(message);
+    this.name = "MdqOperationError";
+  }
+}
+function mdq(source) {
+  return new MarkdownQuery(String(source));
+}
+function parseQuery(input) {
+  const segments = [];
+  SEGMENT.lastIndex = 0;
+  while (SEGMENT.lastIndex < input.length) {
+    const at = SEGMENT.lastIndex;
+    const match = SEGMENT.exec(input);
+    if (!match)
+      throw new MdqSelectorError(`Unexpected character "${input[at]}" in selector`, at);
+    const [, prefix, name, negated, contains, quoted, pattern, flags, brackets] = match;
+    if (!SELECTOR_NAMES.has(name) && !/^(?:h|section)[1-6]$/.test(name))
+      throw new MdqSelectorError(`Unknown selector "${name}"`, at + prefix.length);
+    const segment = { selector: name, index: null, slice: null };
+    if (quoted !== undefined) {
+      let mode = "exact";
+      if (contains)
+        mode = "contains";
+      segment.textMatch = { mode, value: quoted.replace(/\\(.)/g, "$1"), negated: Boolean(negated) };
+    }
+    if (pattern !== undefined) {
+      segment.textMatch = { mode: "regex", value: pattern, negated: Boolean(negated), flags };
+    }
+    for (const [, bounds] of brackets.matchAll(/\[([^\]]*)\]/g)) {
+      if (!bounds || !/^-?\d*(?::-?\d*)?$/.test(bounds))
+        continue;
+      if (!bounds.includes(":")) {
+        segment.index = Number.parseInt(bounds, 10);
+        continue;
+      }
+      const [from, to] = bounds.split(":");
+      segment.slice = {};
+      if (from)
+        segment.slice.from = Number.parseInt(from, 10);
+      if (to)
+        segment.slice.to = Number.parseInt(to, 10);
+    }
+    segments.push(segment);
+  }
+  return segments;
+}
+function buildTokenIndex(source) {
+  const { body, offset } = MarkdownEditor.splitFrontmatter(source);
+  const ranges = [];
+  let cursor = offset;
+  for (const token of marked.lexer(body)) {
+    const raw = token.raw || "";
+    if (token.type === "space") {
+      const previous = ranges[ranges.length - 1];
+      if (previous)
+        previous.trailing = { start: cursor, length: raw.length };
+      cursor += raw.length;
+      continue;
+    }
+    ranges.push({ token, start: cursor, length: raw.length });
+    cursor += raw.length;
+  }
+  return ranges;
+}
+var SELECTOR_NAMES = new Set(["section", "heading", "paragraph", "table", "list", "item", "code", "blockquote", "hr", "html", "comment"]);
+var TOKEN_ALIASES = { item: "list_item" };
+var SEGMENT = /(\s*\.?)([A-Za-z]\w*)(?:\((!?)(~?)(?:"((?:[^"\\]|\\.)*)"|\/((?:[^/\\]|\\.)*)\/([a-z]*))\))?((?:\[[^\]]*\])*)\s*/y;
+export {
+  parseQuery,
+  mdq,
+  buildTokenIndex,
+  MdqSelectorError,
+  MdqOperationError,
+  MdqError,
+  MarkdownQuery,
+  MarkdownEditor
+};

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "mdq-cli",
+  "version": "0.1.0",
+  "description": "Query and edit markdown with a selector language - jq, for markdown",
+  "license": "MIT",
+  "type": "module",
+  "main": "./index.js",
+  "types": "./types/query.d.ts",
+  "exports": {
+    ".": {
+      "types": "./types/query.d.ts",
+      "import": "./index.js"
+    }
+  },
+  "bin": {
+    "mdq": "./bin/mdq.js"
+  },
+  "files": [
+    "index.js",
+    "types/",
+    "bin/",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "keywords": [
+    "markdown",
+    "query",
+    "selector",
+    "cli",
+    "jq",
+    "frontmatter",
+    "marked",
+    "edit"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/testomatio/explorbot",
+    "directory": "src/utils/mdq"
+  },
+  "dependencies": {
+    "marked": "^16.2.0",
+    "yaml": "^2.8.3",
+    "commander": "^14.0.1"
+  }
+}

package/types/edit.d.ts

@@ -0,0 +1,24 @@
+import type { MatchedRange } from './query.ts';
+export declare class MarkdownEditor {
+    private source;
+    constructor(source: string);
+    static splitFrontmatter(source: string): FrontmatterSplit;
+    static blockEnd(range: MatchedRange): number;
+    static table(headers: string[], rows: string[][], align: (string | null)[]): string;
+    static item(listRaw: string, text: string): string;
+    static entries(text: string, key: string, value: string | null, isBlockquote: boolean): string;
+    remove(ranges: MatchedRange[]): string;
+    replace(ranges: MatchedRange[], render: (range: MatchedRange, index: number) => string): string;
+    insert(offsets: number[], markdown: string): string;
+    frontmatter(): Record<string, unknown>;
+    setFrontmatter(key: string, value: unknown): string;
+    private static spliceIn;
+    private static dashes;
+    private static entryKey;
+    private dedupe;
+}
+export interface FrontmatterSplit {
+    raw: string;
+    body: string;
+    offset: number;
+}