@uniweb/semantic-parser 1.0.0 → 1.0.1

package/package.json

@@ -1,41 +1,41 @@
 {
-    "name": "@uniweb/semantic-parser",
-    "version": "1.0.0",
-    "description": "Semantic parser for ProseMirror/TipTap content structures",
-    "type": "module",
-    "main": "./src/index.js",
-    "exports": {
-        ".": "./src/index.js",
-        "./mappers": "./src/mappers/index.js",
-        "./mappers/*": "./src/mappers/*.js"
-    },
-    "scripts": {
-        "test": "NODE_OPTIONS=--experimental-vm-modules jest",
-        "test-report": "NODE_OPTIONS=--experimental-vm-modules jest --json > test-results.json 2>&1",
-        "test:groups": "NODE_OPTIONS=--experimental-vm-modules jest tests/processors/groups.test.js"
-    },
-    "keywords": [
-        "prosemirror",
-        "tiptap",
-        "parser",
-        "semantic",
-        "content"
-    ],
-    "author": "Proximify Inc.",
-    "license": "GPL-3.0-or-later",
-    "devDependencies": {
-        "jest": "^29.7.0"
-    },
-    "repository": {
-        "type": "git",
-        "url": "git+https://github.com/uniweb/semantic-parser.git"
-    },
-    "bugs": {
-        "url": "https://github.com/uniweb/semantic-parser/issues"
-    },
-    "homepage": "https://github.com/uniweb/semantic-parser#readme",
-    "directories": {
-        "doc": "docs",
-        "test": "tests"
-    }
-}
+  "name": "@uniweb/semantic-parser",
+  "version": "1.0.1",
+  "description": "Semantic parser for ProseMirror/TipTap content structures",
+  "type": "module",
+  "main": "./src/index.js",
+  "exports": {
+    ".": "./src/index.js",
+    "./mappers": "./src/mappers/index.js",
+    "./mappers/*": "./src/mappers/*.js"
+  },
+  "keywords": [
+    "prosemirror",
+    "tiptap",
+    "parser",
+    "semantic",
+    "content"
+  ],
+  "author": "Proximify Inc.",
+  "license": "GPL-3.0-or-later",
+  "devDependencies": {
+    "jest": "^29.7.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/uniweb/semantic-parser.git"
+  },
+  "bugs": {
+    "url": "https://github.com/uniweb/semantic-parser/issues"
+  },
+  "homepage": "https://github.com/uniweb/semantic-parser#readme",
+  "directories": {
+    "doc": "docs",
+    "test": "tests"
+  },
+  "scripts": {
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest",
+    "test-report": "NODE_OPTIONS=--experimental-vm-modules jest --json > test-results.json 2>&1",
+    "test:groups": "NODE_OPTIONS=--experimental-vm-modules jest tests/processors/groups.test.js"
+  }
+}

package/src/processors_old/byType.js

@@ -0,0 +1,129 @@
+/**
+ * Organize content elements by their type while preserving context
+ * @param {Array} sequence Flat sequence of elements
+ * @returns {Object} Content organized by type
+ */
+function processByType(sequence) {
+  const collections = {
+    headings: [],
+    paragraphs: [],
+    images: {
+      background: [],
+      content: [],
+      gallery: [],
+      icon: [],
+    },
+    lists: [],
+    dividers: [],
+    metadata: {
+      totalElements: sequence.length,
+      dominantType: null,
+      hasMedia: false,
+    },
+  };
+
+  // Track type frequencies for metadata
+  const typeFrequency = new Map();
+
+  sequence.forEach((element, index) => {
+    // Track element type frequency
+    typeFrequency.set(element.type, (typeFrequency.get(element.type) || 0) + 1);
+
+    // Add context information
+    const context = getElementContext(sequence, index);
+    const enrichedElement = { ...element, context };
+
+    // Process element based on type
+    switch (element.type) {
+      case "heading":
+        collections.headings.push(enrichedElement);
+        break;
+
+      case "paragraph":
+        collections.paragraphs.push(enrichedElement);
+        break;
+
+      case "image": {
+        const role = element.role || "content";
+        if (!collections.images[role]) {
+          collections.images[role] = [];
+        }
+        collections.images[role].push(enrichedElement);
+        collections.metadata.hasMedia = true;
+        break;
+      }
+
+      case "list":
+        collections.lists.push(enrichedElement);
+        break;
+
+      case "divider":
+        collections.dividers.push(enrichedElement);
+        break;
+    }
+  });
+
+  // Calculate dominant type
+  let maxFrequency = 0;
+  typeFrequency.forEach((frequency, type) => {
+    if (frequency > maxFrequency) {
+      maxFrequency = frequency;
+      collections.metadata.dominantType = type;
+    }
+  });
+
+  // Add helper methods
+  addCollectionHelpers(collections);
+
+  return collections;
+}
+
+/**
+ * Get context information for an element
+ */
+function getElementContext(sequence, position) {
+  const context = {
+    position,
+    previousElement: position > 0 ? sequence[position - 1] : null,
+    nextElement: position < sequence.length - 1 ? sequence[position + 1] : null,
+    nearestHeading: null,
+  };
+
+  // Find nearest preceding heading
+  for (let i = position - 1; i >= 0; i--) {
+    if (sequence[i].type === "heading") {
+      context.nearestHeading = sequence[i];
+      break;
+    }
+  }
+
+  return context;
+}
+
+/**
+ * Add helper methods to collections
+ */
+function addCollectionHelpers(collections) {
+  // Get headings of specific level
+  collections.getHeadingsByLevel = function (level) {
+    return this.headings.filter((h) => h.level === level);
+  };
+
+  // Get elements by heading context
+  collections.getElementsByHeadingContext = function (headingFilter) {
+    const allElements = [
+      ...this.paragraphs,
+      ...Object.values(this.images).flat(),
+      ...this.lists,
+    ];
+
+    return allElements.filter(
+      (el) =>
+        el.context?.nearestHeading && headingFilter(el.context.nearestHeading)
+    );
+  };
+}
+
+module.exports = {
+  processByType,
+};

package/src/processors_old/groups.js

@@ -0,0 +1,240 @@
+/**
+ * Transform a sequence into content groups with semantic structure
+ * @param {Array} sequence Flat sequence of elements
+ * @returns {Object} Content organized into groups with identified main content
+ */
+function processGroups(sequence) {
+  const result = {
+    main: null,
+    items: [],
+    metadata: {
+      dividerMode: false,
+      groups: 0,
+    },
+  };
+
+  if (!sequence.length) return result;
+
+  // Check if using divider mode
+  result.metadata.dividerMode = sequence.some((el) => el.type === "divider");
+
+  // Split sequence into raw groups
+  const groups = result.metadata.dividerMode
+    ? splitByDividers(sequence)
+    : splitByHeadings(sequence);
+
+  // Process each group's structure
+  const processedGroups = groups.map(processGroupContent);
+
+  // Special handling for first group in divider mode
+  if (result.metadata.dividerMode && groups.startsWithDivider) {
+    result.items = processedGroups;
+  } else {
+    // Organize into main content and items
+    const shouldBeMain = identifyMainContent(processedGroups);
+    if (shouldBeMain) {
+      result.main = processedGroups[0];
+      result.items = processedGroups.slice(1);
+    } else {
+      result.items = processedGroups;
+    }
+  }
+
+  // result.metadata.groups = processedGroups.length;
+  return result;
+}
+
+/**
+ * Split sequence into groups using dividers
+ */
+function splitByDividers(sequence) {
+  const groups = [];
+  let currentGroup = [];
+  let startsWithDivider = false;
+
+  // Check if content effectively starts with divider (ignoring whitespace etc)
+  for (let i = 0; i < sequence.length; i++) {
+    const element = sequence[i];
+
+    if (element.type === "divider") {
+      if (currentGroup.length === 0 && groups.length === 0) {
+        startsWithDivider = true;
+      } else if (currentGroup.length > 0) {
+        groups.push(currentGroup);
+        currentGroup = [];
+      }
+    } else {
+      currentGroup.push(element);
+    }
+  }
+
+  if (currentGroup.length > 0) {
+    groups.push(currentGroup);
+  }
+
+  groups.startsWithDivider = startsWithDivider;
+  return groups;
+}
+
+/**
+ * Split sequence into groups using heading patterns
+ */
+function splitByHeadings(sequence) {
+  const groups = [];
+  let currentGroup = [];
+  let isPreOpened = false;
+
+  // Consider if current group is pre opened (only has banner or pretitle)
+  // before starting a new group.
+  const startGroup = (preOpen) => {
+    if (currentGroup.length && !isPreOpened) {
+      groups.push(currentGroup);
+      currentGroup = [];
+    }
+    isPreOpened = preOpen;
+  };
+
+  for (let i = 0; i < sequence.length; i++) {
+    // Only allow a banner for the first group
+    if (!groups.length && isBannerImage(sequence, i)) {
+      startGroup(true); // pre open a new group
+      currentGroup.push(sequence[i]);
+      i++; // move to known next element (it will be a heading)
+    }
+
+    // Handle special pretitle case before consuming all consecutive
+    // headings with increasing levels
+    if (isPreTitle(sequence, i)) {
+      startGroup(true); // pre open a new group
+      currentGroup.push(sequence[i]);
+      i++; // move to known next element (it will be a heading)
+    }
+
+    const element = sequence[i];
+
+    if (element.type === "heading") {
+      const headings = readHeadingGroup(sequence, i);
+      startGroup(false);
+
+      // Add headings to the current group
+      currentGroup.push(...headings);
+      i += headings.length - 1; // skip all the added headings
+    } else {
+      currentGroup.push(element);
+    }
+  }
+
+  if (currentGroup.length > 0) {
+    groups.push(currentGroup);
+  }
+
+  return groups;
+}
+
+/**
+ * Check if this is a pretitle (eg, H3 followed by H1/H2)
+ */
+function isPreTitle(sequence, i) {
+  return (
+    i + 1 < sequence.length &&
+    sequence[i].type === "heading" &&
+    sequence[i + 1].type === "heading" &&
+    sequence[i].level > sequence[i + 1].level
+  );
+
+  // return (
+  //   i + 1 < sequence.length &&
+  //   sequence[i].type === "heading" &&
+  //   sequence[i].level === 3 &&
+  //   sequence[i + 1].type === "heading" &&
+  //   sequence[i + 1].level <= 2
+  // );
+}
+
+function isBannerImage(sequence, i) {
+  return (
+    i + 1 < sequence.length &&
+    sequence[i].type === "image" &&
+    sequence[i + 1].type === "heading"
+  );
+}
+
+/**
+ * Eagerly consume all consecutive headings with increasing levels
+ * and return them as an array.
+ */
+function readHeadingGroup(sequence, i) {
+  const elements = [sequence[i]];
+  for (i++; i < sequence.length; i++) {
+    const element = sequence[i];
+
+    if (element.type === "heading" && element.level > sequence[i - 1].level) {
+      elements.push(element);
+    }
+  }
+  return elements;
+}
+
+/**
+ * Process a group's content to identify its structure
+ */
+function processGroupContent(elements) {
+  const content = [];
+  const headings = {
+    pretitle: null,
+    title: null,
+    subtitle: null,
+    subsubtitle: null,
+  };
+  const metadata = {
+    level: null,
+    contentTypes: new Set(),
+  };
+
+  for (let i = 0; i < elements.length; i++) {
+    if (isPreTitle(elements, i)) {
+      headings.pretitle = elements[i];
+      i++; // move to known next heading (H1 or h2)
+    }
+
+    const element = elements[i];
+
+    if (element.type === "heading") {
+      metadata.level ??= element.level;
+
+      if (!headings.title) {
+        headings.title = element;
+      } else if (!headings.subtitle) {
+        headings.subtitle = element;
+      } else if (!headings.subsubtitle) {
+        headings.subsubtitle = element;
+      }
+      // What do we do if more headings?
+    } else {
+      content.push(element);
+      metadata.contentTypes.add(element.type);
+    }
+  }
+
+  return { headings, content, metadata };
+}
+
+/**
+ * Determine if the first group should be treated as main content
+ */
+function identifyMainContent(groups) {
+  if (groups.length === 0) return false;
+
+  // Single group is main content
+  if (groups.length === 1) return true;
+
+  // First group should be more important (lower level) than second to be main
+  const first = groups[0].metadata.level;
+  const second = groups[1].metadata.level;
+
+  return first ? !second || first < second : false;
+}
+
+module.exports = {
+  processGroups,
+};

package/src/processors_old/sequence.js

@@ -0,0 +1,140 @@
+/**
+ * Process a ProseMirror/TipTap document into a flat sequence
+ * @param {Object} doc ProseMirror document
+ * @returns {Array} Sequence of content elements
+ */
+function processSequence(doc) {
+  const sequence = [];
+  processNode(doc, sequence);
+  return sequence;
+}
+
+function processNode(node, sequence) {
+  // Special handling for root doc node
+  if (node.type === "doc") {
+    node.content?.forEach((child) => processNode(child, sequence));
+    return;
+  }
+
+  // Create element based on node type
+  const element = createSequenceElement(node);
+
+  if (element) {
+    // Process marks from node or content
+    if (node.marks?.length || hasMarkedContent(node)) {
+      element.marks = collectMarks(node);
+    }
+
+    sequence.push(element);
+  }
+
+  // Process children if they exist and not already processed
+  if (node.content && !element?.items) {
+    node.content.forEach((child) => processNode(child, sequence));
+  }
+}
+
+function createSequenceElement(node) {
+  switch (node.type) {
+    case "heading":
+      return {
+        type: "heading",
+        level: node.attrs.level,
+        content: getTextContent(node),
+      };
+
+    case "paragraph":
+      return {
+        type: "paragraph",
+        content: getTextContent(node),
+      };
+
+    case "image":
+      return {
+        type: "image",
+        src: node.attrs.src,
+        alt: node.attrs.alt,
+        role: node.attrs.role || "content",
+      };
+
+    case "bulletList":
+    case "orderedList":
+      return {
+        type: "list",
+        style: node.type === "bulletList" ? "bullet" : "ordered",
+        items: processListItems(node),
+      };
+
+    case "listItem":
+      return {
+        type: "listItem",
+        content: getTextContent(node),
+      };
+
+    case "horizontalRule":
+      return {
+        type: "divider",
+      };
+
+    case "text":
+      return null;
+
+    default:
+      return {
+        type: node.type,
+        content: getTextContent(node),
+      };
+  }
+}
+
+function getTextContent(node) {
+  if (!node.content) return "";
+  return node.content.reduce((text, child) => {
+    if (child.type === "text") {
+      return text + child.text;
+    }
+    return text + getTextContent(child);
+  }, "");
+}
+
+function hasMarkedContent(node) {
+  if (!node.content) return false;
+  return node.content.some(
+    (child) => child.marks?.length || hasMarkedContent(child)
+  );
+}
+
+function collectMarks(node) {
+  const marks = new Set();
+
+  if (node.marks) {
+    node.marks.forEach((mark) => marks.add(mark.type));
+  }
+
+  if (node.content) {
+    node.content.forEach((child) => {
+      collectMarks(child).forEach((mark) => marks.add(mark));
+    });
+  }
+
+  return Array.from(marks);
+}
+
+function processListItems(node) {
+  const items = [];
+  node.content?.forEach((item) => {
+    if (item.type === "listItem") {
+      items.push({
+        content: getTextContent(item),
+        items: item.content
+          ?.filter((child) => child.type.endsWith("List"))
+          .flatMap((list) => processListItems(list)),
+      });
+    }
+  });
+  return items;
+}
+
+module.exports = {
+  processSequence,
+};