@tryghost/algolia-fragmenter 0.2.10 → 0.3.0

package/README.md

@@ -19,14 +19,18 @@ pnpm add @tryghost/algolia-fragmenter
 Convert Ghost Content API posts, then reduce the resulting records into fragments:
 
 ```js
-const transforms = require('@tryghost/algolia-fragmenter');
+import {fragmentTransformer, transformToAlgoliaObject} from '@tryghost/algolia-fragmenter';
 
-const records = transforms.transformToAlgoliaObject(posts);
-const fragments = records.reduce(transforms.fragmentTransformer, []);
+const records = transformToAlgoliaObject(posts);
+const fragments = records.reduce(fragmentTransformer, []);
 ```
 
 `transformToAlgoliaObject` accepts an optional array of post slugs to exclude as its second argument. `fragmentTransformer` is designed to be passed directly to `Array#reduce`.
 
+Both operations are deprecated compatibility wrappers. They remain available with their existing output while a deeper record-building API is introduced separately.
+
+This package is ESM-only and requires Node.js 24 or later.
+
 ## Development
 
 Install dependencies from the repository root with `pnpm install`. From the root, run this package's tests and lint checks with:

package/lib/index.d.mts

@@ -0,0 +1,16 @@
+export type GhostContent = Readonly<Record<string, unknown>>;
+export type AlgoliaRecord = Record<string, unknown>;
+/**
+ * @deprecated Retained for compatibility while the deep record-building API is introduced.
+ */
+export declare const fragmentTransformer: (recordAccumulator: AlgoliaRecord[], ghostContent: AlgoliaRecord) => AlgoliaRecord[];
+/**
+ * @deprecated Retained for compatibility while the deep record-building API is introduced.
+ */
+export declare const transformToAlgoliaObject: (posts: readonly GhostContent[], ignoreSlugs?: readonly string[]) => AlgoliaRecord[];
+declare const _default: {
+    fragmentTransformer: typeof fragmentTransformer;
+    transformToAlgoliaObject: typeof transformToAlgoliaObject;
+};
+export default _default;
+//# sourceMappingURL=index.d.mts.map

package/lib/index.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.mts","sourceRoot":"","sources":["../src/index.mts"],"names":[],"mappings":"AAEA,MAAM,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAE7D,MAAM,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAoEpD;;GAEG;AACH,eAAO,MAAM,mBAAmB,sBACT,aAAa,EAAE,gBACpB,aAAa,KAC5B,aAAa,EASf,CAAC;AAqBF;;GAEG;AACH,eAAO,MAAM,wBAAwB,UAC1B,SAAS,YAAY,EAAE,gBAChB,SAAS,MAAM,EAAE,KAChC,aAAa,EAqBf,CAAC"}

package/lib/index.mjs

@@ -0,0 +1,80 @@
+import { extract } from '@tryghost/algolia-html-extractor';
+const createLegacyFragment = (fragment) => ({
+    html: fragment.html,
+    content: fragment.text,
+    headings: [...fragment.headingPath],
+    anchor: fragment.anchor,
+    sourceTag: fragment.sourceTag,
+    customRanking: {
+        position: fragment.position,
+        heading: fragment.headingRank
+    }
+});
+const reduceFragmentsUnderHeadings = (groups, fragment) => {
+    const existingGroup = groups.find(group => group.anchor === fragment.anchor);
+    if (existingGroup === undefined) {
+        groups.push(fragment);
+        return groups;
+    }
+    existingGroup.html += fragment.sourceTag === 'pre' ? ` ${fragment.content}` : fragment.html;
+    existingGroup.content += ` ${fragment.content}`;
+    return groups;
+};
+const toAlgoliaRecord = (ghostContent, fragment, index) => {
+    const { content: _content, sourceTag: _sourceTag, ...groupedFragment } = fragment;
+    const url = fragment.anchor === null ? ghostContent.url : `${ghostContent.url}#${fragment.anchor}`;
+    return {
+        ...ghostContent,
+        ...groupedFragment,
+        url,
+        objectID: `${ghostContent.objectID}_${index}`
+    };
+};
+/**
+ * @deprecated Retained for compatibility while the deep record-building API is introduced.
+ */
+export const fragmentTransformer = (recordAccumulator, ghostContent) => {
+    const groupedFragments = extract(ghostContent.html)
+        .map(createLegacyFragment)
+        .reduce(reduceFragmentsUnderHeadings, []);
+    const records = groupedFragments.map((fragment, index) => toAlgoliaRecord(ghostContent, fragment, index));
+    return [...recordAccumulator, ...records];
+};
+const projectLegacyRelations = (value, fieldName) => {
+    const relations = value;
+    if (!relations?.length) {
+        return [];
+    }
+    if (typeof relations.forEach !== 'function') {
+        throw new TypeError(`post.${fieldName}.forEach is not a function`);
+    }
+    const projected = [];
+    relations.forEach(relation => {
+        projected.push({ name: relation.name, slug: relation.slug });
+    });
+    return projected;
+};
+/**
+ * @deprecated Retained for compatibility while the deep record-building API is introduced.
+ */
+export const transformToAlgoliaObject = (posts, ignoreSlugs) => {
+    const algoliaObjects = [];
+    for (const post of posts) {
+        if (ignoreSlugs?.some(slug => slug === post.slug)) {
+            continue;
+        }
+        algoliaObjects.push({
+            objectID: post.id,
+            slug: post.slug,
+            url: post.url,
+            html: post.html,
+            image: post.feature_image,
+            title: post.title,
+            tags: projectLegacyRelations(post.tags, 'tags'),
+            authors: projectLegacyRelations(post.authors, 'authors')
+        });
+    }
+    return algoliaObjects;
+};
+export default { fragmentTransformer, transformToAlgoliaObject };
+//# sourceMappingURL=index.mjs.map

package/lib/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","sourceRoot":"","sources":["../src/index.mts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAA0C,MAAM,kCAAkC,CAAC;AA4BlG,MAAM,oBAAoB,GAAG,CAAC,QAA4C,EAAkB,EAAE,CAAC,CAAC;IAC5F,IAAI,EAAE,QAAQ,CAAC,IAAI;IACnB,OAAO,EAAE,QAAQ,CAAC,IAAI;IACtB,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC;IACnC,MAAM,EAAE,QAAQ,CAAC,MAAM;IACvB,SAAS,EAAE,QAAQ,CAAC,SAAS;IAC7B,aAAa,EAAE;QACX,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,OAAO,EAAE,QAAQ,CAAC,WAAW;KAChC;CACJ,CAAC,CAAC;AAEH,MAAM,4BAA4B,GAAG,CACjC,MAAwB,EACxB,QAAwB,EACR,EAAE;IAClB,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC7E,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtB,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,aAAa,CAAC,IAAI,IAAI,QAAQ,CAAC,SAAS,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC5F,aAAa,CAAC,OAAO,IAAI,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;IAChD,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CACpB,YAA2B,EAC3B,QAAwB,EACxB,KAAa,EACA,EAAE;IACf,MAAM,EAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,eAAe,EAAC,GAAG,QAAQ,CAAC;IAChF,MAAM,GAAG,GACL,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;IAE3F,OAAO;QACH,GAAG,YAAY;QACf,GAAG,eAAe;QAClB,GAAG;QACH,QAAQ,EAAE,GAAG,YAAY,CAAC,QAAQ,IAAI,KAAK,EAAE;KAChD,CAAC;AACN,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAC/B,iBAAkC,EAClC,YAA2B,EACZ,EAAE;IACjB,MAAM,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC,IAAc,CAAC;SACxD,GAAG,CAAC,oBAAoB,CAAC;SACzB,MAAM,CAAC,4BAA4B,EAAE,EAAE,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,CACrD,eAAe,CAAC,YAAY,EAAE,QAAQ,EAAE,KAAK,CAAC,CACjD,CAAC;IAEF,OAAO,CAAC,GAAG,iBAAiB,EAAE,GAAG,OAAO,CAAC,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAC3B,KAAc,EACd,SAA6B,EACQ,EAAE;IACvC,MAAM,SAAS,GAAG,KAA6D,CAAC;IAChF,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;QACrB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,IAAI,OAAO,SAAS,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;QAC1C,MAAM,IAAI,SAAS,CAAC,QAAQ,SAAS,4BAA4B,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,SAAS,GAA0C,EAAE,CAAC;IAC5D,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE;QACzB,SAAS,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAC,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IACH,OAAO,SAAS,CAAC;AACrB,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CACpC,KAA8B,EAC9B,WAA+B,EAChB,EAAE;IACjB,MAAM,cAAc,GAAoB,EAAE,CAAC;IAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,SAAS;QACb,CAAC;QAED,cAAc,CAAC,IAAI,CAAC;YAChB,QAAQ,EAAE,IAAI,CAAC,EAAE;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,aAAa;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,sBAAsB,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC;YAC/C,OAAO,EAAE,sBAAsB,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;SAC3D,CAAC,CAAC;IACP,CAAC;IAED,OAAO,cAAc,CAAC;AAC1B,CAAC,CAAC;AAEF,eAAe,EAAC,mBAAmB,EAAE,wBAAwB,EAAC,CAAC","sourcesContent":["import {extract, type ExtractedTagName, type HeadingRank} from '@tryghost/algolia-html-extractor';\n\nexport type GhostContent = Readonly<Record<string, unknown>>;\n\nexport type AlgoliaRecord = Record<string, unknown>;\n\ntype GhostRelation = Readonly<Record<string, unknown>>;\n\ntype LegacyRelationCollection = {\n    length: number;\n    forEach(callback: (relation: GhostRelation) => void): void;\n};\n\ntype GroupedFragment = {\n    html: string;\n    headings: string[];\n    anchor: string | null;\n    customRanking: {\n        position: number;\n        heading: HeadingRank;\n    };\n};\n\ntype LegacyFragment = GroupedFragment & {\n    content: string;\n    sourceTag: ExtractedTagName;\n};\n\nconst createLegacyFragment = (fragment: ReturnType<typeof extract>[number]): LegacyFragment => ({\n    html: fragment.html,\n    content: fragment.text,\n    headings: [...fragment.headingPath],\n    anchor: fragment.anchor,\n    sourceTag: fragment.sourceTag,\n    customRanking: {\n        position: fragment.position,\n        heading: fragment.headingRank\n    }\n});\n\nconst reduceFragmentsUnderHeadings = (\n    groups: LegacyFragment[],\n    fragment: LegacyFragment\n): LegacyFragment[] => {\n    const existingGroup = groups.find(group => group.anchor === fragment.anchor);\n    if (existingGroup === undefined) {\n        groups.push(fragment);\n        return groups;\n    }\n\n    existingGroup.html += fragment.sourceTag === 'pre' ? ` ${fragment.content}` : fragment.html;\n    existingGroup.content += ` ${fragment.content}`;\n    return groups;\n};\n\nconst toAlgoliaRecord = (\n    ghostContent: AlgoliaRecord,\n    fragment: LegacyFragment,\n    index: number\n): AlgoliaRecord => {\n    const {content: _content, sourceTag: _sourceTag, ...groupedFragment} = fragment;\n    const url =\n        fragment.anchor === null ? ghostContent.url : `${ghostContent.url}#${fragment.anchor}`;\n\n    return {\n        ...ghostContent,\n        ...groupedFragment,\n        url,\n        objectID: `${ghostContent.objectID}_${index}`\n    };\n};\n\n/**\n * @deprecated Retained for compatibility while the deep record-building API is introduced.\n */\nexport const fragmentTransformer = (\n    recordAccumulator: AlgoliaRecord[],\n    ghostContent: AlgoliaRecord\n): AlgoliaRecord[] => {\n    const groupedFragments = extract(ghostContent.html as string)\n        .map(createLegacyFragment)\n        .reduce(reduceFragmentsUnderHeadings, []);\n    const records = groupedFragments.map((fragment, index) =>\n        toAlgoliaRecord(ghostContent, fragment, index)\n    );\n\n    return [...recordAccumulator, ...records];\n};\n\nconst projectLegacyRelations = (\n    value: unknown,\n    fieldName: 'tags' | 'authors'\n): Array<{name: unknown; slug: unknown}> => {\n    const relations = value as Partial<LegacyRelationCollection> | null | undefined;\n    if (!relations?.length) {\n        return [];\n    }\n    if (typeof relations.forEach !== 'function') {\n        throw new TypeError(`post.${fieldName}.forEach is not a function`);\n    }\n\n    const projected: Array<{name: unknown; slug: unknown}> = [];\n    relations.forEach(relation => {\n        projected.push({name: relation.name, slug: relation.slug});\n    });\n    return projected;\n};\n\n/**\n * @deprecated Retained for compatibility while the deep record-building API is introduced.\n */\nexport const transformToAlgoliaObject = (\n    posts: readonly GhostContent[],\n    ignoreSlugs?: readonly string[]\n): AlgoliaRecord[] => {\n    const algoliaObjects: AlgoliaRecord[] = [];\n\n    for (const post of posts) {\n        if (ignoreSlugs?.some(slug => slug === post.slug)) {\n            continue;\n        }\n\n        algoliaObjects.push({\n            objectID: post.id,\n            slug: post.slug,\n            url: post.url,\n            html: post.html,\n            image: post.feature_image,\n            title: post.title,\n            tags: projectLegacyRelations(post.tags, 'tags'),\n            authors: projectLegacyRelations(post.authors, 'authors')\n        });\n    }\n\n    return algoliaObjects;\n};\n\nexport default {fragmentTransformer, transformToAlgoliaObject};\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tryghost/algolia-fragmenter",
-  "version": "0.2.10",
+  "version": "0.3.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/TryGhost/algolia.git",
@@ -11,21 +11,35 @@
   "engines": {
     "node": ">=24"
   },
-  "main": "index.js",
+  "type": "module",
+  "types": "./lib/index.d.mts",
+  "exports": {
+    ".": {
+      "types": "./lib/index.d.mts",
+      "import": "./lib/index.mjs"
+    }
+  },
   "files": [
-    "index.js",
     "lib"
   ],
   "publishConfig": {
     "access": "public"
   },
   "dependencies": {
-    "algolia-html-extractor": "0.0.1"
+    "@tryghost/algolia-html-extractor": "^0.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "24.13.3",
+    "typescript": "7.0.2"
   },
   "scripts": {
     "dev": "echo \"Implement me!\"",
+    "prebuild": "pnpm --filter @tryghost/algolia-html-extractor build",
+    "pretest": "pnpm build",
     "test": "NODE_ENV=testing vitest run --root .",
+    "typecheck": "tsc --project tsconfig.json",
+    "build": "tsc --project tsconfig.build.json",
     "lint": "oxlint --quiet . && oxfmt --check .",
-    "posttest": "pnpm lint"
+    "posttest": "pnpm typecheck && pnpm lint"
   }
 }

package/index.js

@@ -1 +0,0 @@
-module.exports = require('./lib/transformer');

package/lib/transformer.js

@@ -1,116 +0,0 @@
-const HtmlExtractor = require(`algolia-html-extractor`);
-const Extractor = new HtmlExtractor();
-
-/**
- * Utility function, takes the output of HTML Extractor, and reduces it back down
- * So that there is a group of HTML/content per heading
- *
- * @param {Array} accumulator
- * @param {Object} fragment
- */
-const reduceFragmentsUnderHeadings = (accumulator, fragment) => {
-    const existingFragment = accumulator.find(existing => existing.anchor === fragment.anchor);
-
-    if (existingFragment) {
-        // Merge our fragments together
-        if (fragment.node && fragment.node.tagName === `PRE`) {
-            // For pre-tags, we don't keep all the markup
-            existingFragment.html += ` ${fragment.content}`; // keep a space
-            existingFragment.content += ` ${fragment.content}`; // keep a space
-        } else {
-            existingFragment.html += fragment.html;
-            existingFragment.content += ` ${fragment.content}`; // keep a space
-        }
-    } else {
-        // If we don't already have a matching fragment with this anchor, add it
-        accumulator.push(fragment);
-    }
-
-    return accumulator;
-};
-
-/**
- * Fragment Transformer
- * breaks down large HTML strings into sensible fragments based on headings
- */
-module.exports.fragmentTransformer = (recordAccumulator, node) => {
-    let htmlFragments = Extractor
-        // These are the top-level HTML elements that we keep - this results in a lot of fragments
-        .run(node.html, {cssSelector: `p,pre,td,li`})
-        // Use the utility function to merge fragments so that there is one-per-heading
-        .reduce(reduceFragmentsUnderHeadings, []);
-
-    // convert our fragments for this node into valid objects, and merge int the
-    const records = htmlFragments.reduce((fragmentAccumulator, fragment, index) => {
-        // Don't need a reference to the html node type
-        delete fragment.node;
-        // For now at least, we're not going to index the content string
-        // The HTML string is already very long, and there are size limits
-        delete fragment.content;
-        // If we have an anchor, change the URL to be a deep link
-        if (fragment.anchor) {
-            fragment.url = `${node.url}#${fragment.anchor}`;
-        }
-
-        let objectID = `${node.objectID}_${index}`;
-
-        // TODO: switch this on in verbose mode only
-        // // If fragments are too long, we need this to see which fragment it was
-
-        return [...fragmentAccumulator, {...node, ...fragment, objectID: objectID}];
-    }, []);
-
-    return [...recordAccumulator, ...records];
-};
-
-module.exports._testReduceFragmentsUnderHeadings = reduceFragmentsUnderHeadings;
-
-/**
- * Algolia Object Transformer
- * takes a Ghost post and selects the properties needed to send to Algolia
- *
- *  @param {Array} posts
- */
-module.exports.transformToAlgoliaObject = (posts, ignoreSlugs) => {
-    const algoliaObjects = [];
-
-    posts.map(post => {
-        // Define the properties we need for Algolia
-        const algoliaPost = {
-            objectID: post.id,
-            slug: post.slug,
-            url: post.url,
-            html: post.html,
-            image: post.feature_image,
-            title: post.title,
-            tags: [],
-            authors: []
-        };
-
-        // If we have an array of slugs to ignore, and the current
-        // post slug is in that list, skip this loop iteration
-        if (ignoreSlugs) {
-            if (ignoreSlugs.includes(post.slug)) {
-                return false;
-            }
-        }
-
-        if (post.tags && post.tags.length) {
-            post.tags.forEach(tag => {
-                algoliaPost.tags.push({name: tag.name, slug: tag.slug});
-            });
-        }
-
-        if (post.authors && post.authors.length) {
-            post.authors.forEach(author => {
-                algoliaPost.authors.push({name: author.name, slug: author.slug});
-            });
-        }
-
-        algoliaObjects.push(algoliaPost);
-
-        return algoliaPost;
-    });
-
-    return algoliaObjects;
-};