@memberjunction/ai-segmentation 0.0.0 → 5.51.0

package/dist/generic/SemanticTextSegmenter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SemanticTextSegmenter.js","sourceRoot":"","sources":["../../src/generic/SemanticTextSegmenter.ts"],"names":[],"mappings":"AAAA;;;;GAIG;;;;;;;AAEH,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAY,MAAM,sBAAsB,CAAC;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGpE,0DAA0D;AAC1D,MAAM,CAAC,MAAM,2BAA2B,GAAG,cAAc,CAAC;AAE1D,8EAA8E;AAC9E,MAAM,CAAC,MAAM,iCAAiC,GAAG,+BAA+B,CAAC;AAqCjF;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEI,IAAM,qBAAqB,GAA3B,MAAM,qBAAsB,SAAQ,aAAa;IACpD,IAAW,GAAG;QACV,OAAO,2BAA2B,CAAC;IACvC,CAAC;IAED,IAAW,mBAAmB;QAC1B,OAAO,CAAC,MAAM,CAAC,CAAC;IACpB,CAAC;IAES,KAAK,CAAC,WAAW,CAAC,MAA2D;QACnF,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,EAAE,CAAC;QACd,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,EAAE,eAAe,IAAI,GAAG,CAAC;QACzD,IAAI,IAAI,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC;YAC9D,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACtD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAChE,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzC,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED,gDAAgD;IAChD,oBAAoB;IACpB,gDAAgD;IAEhD,kEAAkE;IAC1D,WAAW,CAAC,IAAY,EAAE,OAAyC;QACvE,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,UAAU,CAAC;QACzB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC;YACpB,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YAClD,MAAM,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACvC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,IAAI,GAAG,CAAC,CAAC;IACtD,CAAC;IAED,iDAAiD;IACzC,SAAS,CAAC,MAAuB,EAAE,IAAY,EAAE,KAAa,EAAE,GAAW;QAC/E,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;QAC1F,CAAC;IACL,CAAC;IAED,gDAAgD;IAChD,yBAAyB;IACzB,gDAAgD;IAEhD,+EAA+E;IACvE,KAAK,CAAC,iBAAiB,CAC3B,MAAuB,EACvB,MAA2D;QAE3D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,EAAE,UAAU,IAAI,iCAAiC,CAAC;QACnF,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QACrG,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,SAAS,CAAC,mCAAmC,UAAU,wDAAwD,CAAC,CAAC;YACjH,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,cAAc,EAAE,CAAC;QAC1C,YAAY,CAAC,MAAM,GAAG,MAAM,CAAC;QAC7B,YAAY,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QAC9C,YAAY,CAAC,IAAI,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QACvG,YAAY,CAAC,iBAAiB,GAAG,IAAI,CAAC;QACtC,YAAY,CAAC,oBAAoB,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC;QACzD,IAAI,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;YAC1B,YAAY,CAAC,QAAQ,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAChE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,cAAc,EAAE,CAAC,aAAa,CAAmB,YAAY,CAAC,CAAC;QACxF,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAClB,QAAQ,CAAC,mDAAmD,MAAM,CAAC,YAAY,IAAI,eAAe,EAAE,CAAC,CAAC;YACtG,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACnF,CAAC;IAED,wDAAwD;IAChD,YAAY,CAAC,MAAuB,EAAE,OAAyC;QACnF,MAAM,YAAY,GAAG,OAAO,EAAE,iBAAiB,IAAI,GAAG,CAAC;QACvD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3F,CAAC;IAED,sEAAsE;IAC9D,WAAW,CAAC,GAA0C;QAC1D,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;YAC1B,OAAO,GAAG,CAAC;QACf,CAAC;QACD,IAAI,CAAC;YACD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAqB,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACL,QAAQ,CAAC,0DAA0D,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YAC5F,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,kBAAkB,CACtB,QAAiC,EACjC,UAAkB;QAElB,MAAM,GAAG,GAAG,QAAQ,EAAE,UAAU,IAAI,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,OAAO,GAA6C,EAAE,CAAC;QAE7D,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;YACxC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,UAAU,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAClF,SAAS;YACb,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,SAAS,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;YACpD,OAAO,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,gDAAgD;IAChD,mBAAmB;IACnB,gDAAgD;IAEhD,wEAAwE;IAChE,aAAa,CAAC,MAAuB,EAAE,UAAoD;QAC/F,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE;YAClC,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;YAC1F,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;YAC1D,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACnD,OAAO;gBACH,QAAQ,EAAE,MAAM;gBAChB,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI;gBAC1D,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW;gBAClC,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,SAAS;aAChD,CAAC;QACN,CAAC,CAAC,CAAC;IACP,CAAC;IAED,0FAA0F;IAClF,KAAK,CAAC,QAAQ,CAAC,MAA2D;QAC9E,MAAM,UAAU,GAAG,IAAI,uBAAuB,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QACnH,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,SAAS,EAAE,CAAC,CAAC,SAAS;SACzB,CAAC,CAAC,CAAC;IACR,CAAC;CACJ,CAAA;AA7KY,qBAAqB;IADjC,aAAa,CAAC,aAAa,EAAE,2BAA2B,CAAC;GAC7C,qBAAqB,CA6KjC"}

package/dist/generic/StructuralTextSegmenter.d.ts

@@ -0,0 +1,63 @@
+/**
+ * @fileoverview Structure-aware text segmenter — splits on document headings.
+ *
+ * @module @memberjunction/ai-segmentation
+ */
+import { BaseSegmenter } from './BaseSegmenter.js';
+import { ContentModality, RawSegment, SegmentationOptions, SegmentationParams } from './Segmentation.types.js';
+/** Registration key for {@link StructuralTextSegmenter}. */
+export declare const STRUCTURAL_TEXT_SEGMENTER_KEY = "StructuralText";
+/** Options specific to {@link StructuralTextSegmenter}. */
+export interface StructuralTextSegmentationOptions extends SegmentationOptions {
+    /**
+     * Prepend the heading text to each section's body before embedding. Keeps a
+     * section's topic in its own vector, which materially helps retrieval when the
+     * body uses pronouns ("it", "the above"). Default: true.
+     */
+    IncludeHeadingInText?: boolean;
+    /**
+     * Heading syntax to look for. `'auto'` detects markdown vs. HTML from the
+     * content (and honours `params.MimeType` when supplied). Default: `'auto'`.
+     */
+    HeadingSyntax?: 'auto' | 'markdown' | 'html';
+}
+/**
+ * Segments text along its own document structure (markdown `#` headings or HTML
+ * `<h1>`-`<h6>`), emitting one segment per section and preserving the heading
+ * hierarchy as parent/child links.
+ *
+ * This is the recommended default for documents, PDFs-converted-to-text, wiki
+ * pages, and knowledge-base articles. Compared with splitting purely on a token
+ * budget, sections are coherent units of meaning, so each vector represents one
+ * topic instead of an arbitrary window that may straddle two.
+ *
+ * When the content has no headings it degrades gracefully to paragraph segments,
+ * which is still better than fixed windows because paragraph breaks are authored
+ * boundaries. In both cases {@link BaseSegmenter} enforces the token ceiling.
+ */
+export declare class StructuralTextSegmenter extends BaseSegmenter {
+    get Key(): string;
+    get SupportedModalities(): ContentModality[];
+    protected SegmentCore(params: SegmentationParams<StructuralTextSegmentationOptions>): RawSegment[];
+    /** Decide which heading syntax applies to this content. */
+    private resolveSyntax;
+    /** Locate ATX-style markdown headings (`#` through `######`). */
+    private findMarkdownHeadings;
+    /** Locate HTML `<h1>`-`<h6>` headings. */
+    private findHtmlHeadings;
+    /** Remove inline tags from a heading's inner HTML. */
+    private stripTags;
+    /** Build one segment per heading section, plus any preamble, with parent links. */
+    private buildSectionSegments;
+    /** Text preceding the first heading, if any. */
+    private buildPreamble;
+    /** Section body, optionally prefixed with its heading for self-contained context. */
+    private buildSectionText;
+    /** Nearest preceding heading with a lower level number (i.e. the enclosing section). */
+    private findParentHeading;
+    /** Fallback for unstructured text: one segment per paragraph, offsets preserved. */
+    private segmentByParagraph;
+    /** Append a paragraph segment when the slice has content. */
+    private pushParagraph;
+}
+//# sourceMappingURL=StructuralTextSegmenter.d.ts.map

package/dist/generic/StructuralTextSegmenter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"StructuralTextSegmenter.d.ts","sourceRoot":"","sources":["../../src/generic/StructuralTextSegmenter.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE5G,4DAA4D;AAC5D,eAAO,MAAM,6BAA6B,mBAAmB,CAAC;AAE9D,2DAA2D;AAC3D,MAAM,WAAW,iCAAkC,SAAQ,mBAAmB;IAC1E;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAAC;CAChD;AAYD;;;;;;;;;;;;;GAaG;AACH,qBACa,uBAAwB,SAAQ,aAAa;IACtD,IAAW,GAAG,IAAI,MAAM,CAEvB;IAED,IAAW,mBAAmB,IAAI,eAAe,EAAE,CAElD;IAED,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,iCAAiC,CAAC,GAAG,UAAU,EAAE;IAiBlG,2DAA2D;IAC3D,OAAO,CAAC,aAAa;IAWrB,iEAAiE;IACjE,OAAO,CAAC,oBAAoB;IAgB5B,0CAA0C;IAC1C,OAAO,CAAC,gBAAgB;IAgBxB,sDAAsD;IACtD,OAAO,CAAC,SAAS;IAQjB,mFAAmF;IACnF,OAAO,CAAC,oBAAoB;IA6B5B,gDAAgD;IAChD,OAAO,CAAC,aAAa;IAQrB,qFAAqF;IACrF,OAAO,CAAC,gBAAgB;IAcxB,wFAAwF;IACxF,OAAO,CAAC,iBAAiB;IASzB,oFAAoF;IACpF,OAAO,CAAC,kBAAkB;IAc1B,6DAA6D;IAC7D,OAAO,CAAC,aAAa;CAMxB"}

package/dist/generic/StructuralTextSegmenter.js

@@ -0,0 +1,177 @@
+/**
+ * @fileoverview Structure-aware text segmenter — splits on document headings.
+ *
+ * @module @memberjunction/ai-segmentation
+ */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+import { RegisterClass } from '@memberjunction/global';
+import { BaseSegmenter } from './BaseSegmenter.js';
+/** Registration key for {@link StructuralTextSegmenter}. */
+export const STRUCTURAL_TEXT_SEGMENTER_KEY = 'StructuralText';
+/**
+ * Segments text along its own document structure (markdown `#` headings or HTML
+ * `<h1>`-`<h6>`), emitting one segment per section and preserving the heading
+ * hierarchy as parent/child links.
+ *
+ * This is the recommended default for documents, PDFs-converted-to-text, wiki
+ * pages, and knowledge-base articles. Compared with splitting purely on a token
+ * budget, sections are coherent units of meaning, so each vector represents one
+ * topic instead of an arbitrary window that may straddle two.
+ *
+ * When the content has no headings it degrades gracefully to paragraph segments,
+ * which is still better than fixed windows because paragraph breaks are authored
+ * boundaries. In both cases {@link BaseSegmenter} enforces the token ceiling.
+ */
+let StructuralTextSegmenter = class StructuralTextSegmenter extends BaseSegmenter {
+    get Key() {
+        return STRUCTURAL_TEXT_SEGMENTER_KEY;
+    }
+    get SupportedModalities() {
+        return ['text'];
+    }
+    SegmentCore(params) {
+        const text = params.Text ?? '';
+        if (text.trim().length === 0) {
+            return [];
+        }
+        const syntax = this.resolveSyntax(text, params);
+        const headings = syntax === 'html' ? this.findHtmlHeadings(text) : this.findMarkdownHeadings(text);
+        if (headings.length === 0) {
+            return this.segmentByParagraph(text);
+        }
+        return this.buildSectionSegments(text, headings, params.Options);
+    }
+    // ─────────────────────────────────────────────
+    // Heading detection
+    // ─────────────────────────────────────────────
+    /** Decide which heading syntax applies to this content. */
+    resolveSyntax(text, params) {
+        const configured = params.Options?.HeadingSyntax ?? 'auto';
+        if (configured !== 'auto') {
+            return configured;
+        }
+        if (params.MimeType && params.MimeType.toLowerCase().includes('html')) {
+            return 'html';
+        }
+        return /<h[1-6][\s>]/i.test(text) ? 'html' : 'markdown';
+    }
+    /** Locate ATX-style markdown headings (`#` through `######`). */
+    findMarkdownHeadings(text) {
+        const regex = /^(#{1,6})[ \t]+(.+?)[ \t]*$/gm;
+        const matches = [];
+        let match = regex.exec(text);
+        while (match !== null) {
+            matches.push({
+                Level: match[1].length,
+                Title: match[2].trim(),
+                StartOffset: match.index,
+                BodyStart: match.index + match[0].length,
+            });
+            match = regex.exec(text);
+        }
+        return matches;
+    }
+    /** Locate HTML `<h1>`-`<h6>` headings. */
+    findHtmlHeadings(text) {
+        const regex = /<h([1-6])[^>]*>([\s\S]*?)<\/h\1>/gi;
+        const matches = [];
+        let match = regex.exec(text);
+        while (match !== null) {
+            matches.push({
+                Level: Number(match[1]),
+                Title: this.stripTags(match[2]).trim(),
+                StartOffset: match.index,
+                BodyStart: match.index + match[0].length,
+            });
+            match = regex.exec(text);
+        }
+        return matches;
+    }
+    /** Remove inline tags from a heading's inner HTML. */
+    stripTags(html) {
+        return html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ');
+    }
+    // ─────────────────────────────────────────────
+    // Segment construction
+    // ─────────────────────────────────────────────
+    /** Build one segment per heading section, plus any preamble, with parent links. */
+    buildSectionSegments(text, headings, options) {
+        const segments = [];
+        const segmentIndexByHeading = new Map();
+        const preamble = this.buildPreamble(text, headings[0].StartOffset);
+        if (preamble) {
+            segments.push(preamble);
+        }
+        headings.forEach((heading, index) => {
+            const end = index + 1 < headings.length ? headings[index + 1].StartOffset : text.length;
+            const parentHeading = this.findParentHeading(headings, index);
+            segments.push({
+                Modality: 'text',
+                Title: heading.Title,
+                Text: this.buildSectionText(text, heading, end, options),
+                StartOffset: heading.StartOffset,
+                EndOffset: end,
+                ParentIndex: parentHeading === undefined ? undefined : segmentIndexByHeading.get(parentHeading),
+            });
+            segmentIndexByHeading.set(index, segments.length - 1);
+        });
+        return segments;
+    }
+    /** Text preceding the first heading, if any. */
+    buildPreamble(text, firstHeadingOffset) {
+        const body = text.slice(0, firstHeadingOffset).trim();
+        if (body.length === 0) {
+            return null;
+        }
+        return { Modality: 'text', Text: body, StartOffset: 0, EndOffset: firstHeadingOffset };
+    }
+    /** Section body, optionally prefixed with its heading for self-contained context. */
+    buildSectionText(text, heading, end, options) {
+        const body = text.slice(heading.BodyStart, end).trim();
+        const includeHeading = options?.IncludeHeadingInText ?? true;
+        if (!includeHeading) {
+            return body;
+        }
+        return body.length > 0 ? `${heading.Title}\n${body}` : heading.Title;
+    }
+    /** Nearest preceding heading with a lower level number (i.e. the enclosing section). */
+    findParentHeading(headings, index) {
+        for (let i = index - 1; i >= 0; i--) {
+            if (headings[i].Level < headings[index].Level) {
+                return i;
+            }
+        }
+        return undefined;
+    }
+    /** Fallback for unstructured text: one segment per paragraph, offsets preserved. */
+    segmentByParagraph(text) {
+        const segments = [];
+        const regex = /\n\s*\n/g;
+        let cursor = 0;
+        let match = regex.exec(text);
+        while (match !== null) {
+            this.pushParagraph(segments, text, cursor, match.index);
+            cursor = match.index + match[0].length;
+            match = regex.exec(text);
+        }
+        this.pushParagraph(segments, text, cursor, text.length);
+        return segments;
+    }
+    /** Append a paragraph segment when the slice has content. */
+    pushParagraph(segments, text, start, end) {
+        const body = text.slice(start, end).trim();
+        if (body.length > 0) {
+            segments.push({ Modality: 'text', Text: body, StartOffset: start, EndOffset: end });
+        }
+    }
+};
+StructuralTextSegmenter = __decorate([
+    RegisterClass(BaseSegmenter, STRUCTURAL_TEXT_SEGMENTER_KEY)
+], StructuralTextSegmenter);
+export { StructuralTextSegmenter };
+//# sourceMappingURL=StructuralTextSegmenter.js.map

package/dist/generic/StructuralTextSegmenter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"StructuralTextSegmenter.js","sourceRoot":"","sources":["../../src/generic/StructuralTextSegmenter.ts"],"names":[],"mappings":"AAAA;;;;GAIG;;;;;;;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhD,4DAA4D;AAC5D,MAAM,CAAC,MAAM,6BAA6B,GAAG,gBAAgB,CAAC;AA2B9D;;;;;;;;;;;;;GAaG;AAEI,IAAM,uBAAuB,GAA7B,MAAM,uBAAwB,SAAQ,aAAa;IACtD,IAAW,GAAG;QACV,OAAO,6BAA6B,CAAC;IACzC,CAAC;IAED,IAAW,mBAAmB;QAC1B,OAAO,CAAC,MAAM,CAAC,CAAC;IACpB,CAAC;IAES,WAAW,CAAC,MAA6D;QAC/E,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,EAAE,CAAC;QACd,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChD,MAAM,QAAQ,GAAG,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;QACnG,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;QACD,OAAO,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IACrE,CAAC;IAED,gDAAgD;IAChD,oBAAoB;IACpB,gDAAgD;IAEhD,2DAA2D;IACnD,aAAa,CAAC,IAAY,EAAE,MAA6D;QAC7F,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,EAAE,aAAa,IAAI,MAAM,CAAC;QAC3D,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;YACxB,OAAO,UAAU,CAAC;QACtB,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACpE,OAAO,MAAM,CAAC;QAClB,CAAC;QACD,OAAO,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;IAC5D,CAAC;IAED,iEAAiE;IACzD,oBAAoB,CAAC,IAAY;QACrC,MAAM,KAAK,GAAG,+BAA+B,CAAC;QAC9C,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC;YACpB,OAAO,CAAC,IAAI,CAAC;gBACT,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;gBACtB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;gBACtB,WAAW,EAAE,KAAK,CAAC,KAAK;gBACxB,SAAS,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;aAC3C,CAAC,CAAC;YACH,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,0CAA0C;IAClC,gBAAgB,CAAC,IAAY;QACjC,MAAM,KAAK,GAAG,oCAAoC,CAAC;QACnD,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC;YACpB,OAAO,CAAC,IAAI,CAAC;gBACT,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACvB,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;gBACtC,WAAW,EAAE,KAAK,CAAC,KAAK;gBACxB,SAAS,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;aAC3C,CAAC,CAAC;YACH,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,sDAAsD;IAC9C,SAAS,CAAC,IAAY;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC9D,CAAC;IAED,gDAAgD;IAChD,uBAAuB;IACvB,gDAAgD;IAEhD,mFAAmF;IAC3E,oBAAoB,CACxB,IAAY,EACZ,QAAwB,EACxB,OAA2C;QAE3C,MAAM,QAAQ,GAAiB,EAAE,CAAC;QAClC,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAkB,CAAC;QAExD,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACnE,IAAI,QAAQ,EAAE,CAAC;YACX,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;QAED,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YAChC,MAAM,GAAG,GAAG,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;YACxF,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC9D,QAAQ,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,MAAM;gBAChB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC;gBACxD,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,SAAS,EAAE,GAAG;gBACd,WAAW,EAAE,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,qBAAqB,CAAC,GAAG,CAAC,aAAa,CAAC;aAClG,CAAC,CAAC;YACH,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,gDAAgD;IACxC,aAAa,CAAC,IAAY,EAAE,kBAA0B;QAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,IAAI,EAAE,CAAC;QACtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAC3F,CAAC;IAED,qFAAqF;IAC7E,gBAAgB,CACpB,IAAY,EACZ,OAAqB,EACrB,GAAW,EACX,OAA2C;QAE3C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACvD,MAAM,cAAc,GAAG,OAAO,EAAE,oBAAoB,IAAI,IAAI,CAAC;QAC7D,IAAI,CAAC,cAAc,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IACzE,CAAC;IAED,wFAAwF;IAChF,iBAAiB,CAAC,QAAwB,EAAE,KAAa;QAC7D,KAAK,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;gBAC5C,OAAO,CAAC,CAAC;YACb,CAAC;QACL,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,oFAAoF;IAC5E,kBAAkB,CAAC,IAAY;QACnC,MAAM,QAAQ,GAAiB,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,UAAU,CAAC;QACzB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC;YACpB,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACvC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACxD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,6DAA6D;IACrD,aAAa,CAAC,QAAsB,EAAE,IAAY,EAAE,KAAa,EAAE,GAAW;QAClF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClB,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;QACxF,CAAC;IACL,CAAC;CACJ,CAAA;AAvKY,uBAAuB;IADnC,aAAa,CAAC,aAAa,EAAE,6BAA6B,CAAC;GAC/C,uBAAuB,CAuKnC"}

package/dist/generic/TranscriptSegmenter.d.ts

@@ -0,0 +1,78 @@
+/**
+ * @fileoverview Transcript-driven segmenter — turns timed cues into chapters.
+ *
+ * @module @memberjunction/ai-segmentation
+ */
+import { BaseSegmenter } from './BaseSegmenter.js';
+import { ContentModality, RawSegment, SegmentationOptions, SegmentationParams } from './Segmentation.types.js';
+/** Registration key for {@link TranscriptSegmenter}. */
+export declare const TRANSCRIPT_SEGMENTER_KEY = "Transcript";
+/** Options specific to {@link TranscriptSegmenter}. */
+export interface TranscriptSegmentationOptions extends SegmentationOptions {
+    /** Maximum wall-clock length of one chapter, in ms. Default: 300000 (5 minutes). */
+    MaxChapterMs?: number;
+    /**
+     * A silence gap at least this long is treated as a chapter boundary — the
+     * cheapest reliable topic-shift signal in a recording. Default: 4000.
+     */
+    BoundaryGapMs?: number;
+    /**
+     * Also emit one child segment per speaker turn inside each chapter. Doubles
+     * embedding cost, so it is opt-in; enable for panel/interview content where
+     * per-speaker retrieval matters. Default: false.
+     */
+    EmitSubChapters?: boolean;
+    /** Modality to stamp on emitted segments when it can't be inferred. Default: `'audio'`. */
+    DefaultModality?: Extract<ContentModality, 'audio' | 'video'>;
+}
+/**
+ * Segments audio/video into **chapters** using its timed transcript.
+ *
+ * A 60-minute session recording embedded as a single vector is a mush vector —
+ * multimodal embedders only see a short window, and no single vector can represent
+ * an hour of distinct topics. This segmenter finds real boundaries first (silence
+ * gaps, speaker changes, a duration ceiling) and emits time-windowed chapters that
+ * each carry **both** a media reference (with `StartMs`/`EndMs`) *and* the
+ * transcript text for that window.
+ *
+ * That dual payload is deliberate: the media reference is what a multimodal model
+ * embeds for retrieval, while the transcript is what an agent can actually read,
+ * what a cross-encoder can rerank, and what keyword search can match.
+ *
+ * Cues come from any ASR source, including MemberJunction's realtime session
+ * capture, which already records per-turn text with speaker labels and timings.
+ */
+export declare class TranscriptSegmenter extends BaseSegmenter {
+    get Key(): string;
+    get SupportedModalities(): ContentModality[];
+    protected SegmentCore(params: SegmentationParams<TranscriptSegmentationOptions>): RawSegment[];
+    /** Cues must be time-ordered before boundary detection; callers may supply any order. */
+    private sortCues;
+    /** Accumulate cues into chapters, breaking on gap, duration, or token pressure. */
+    private groupIntoChapters;
+    /** True when `next` should start a new chapter rather than extend the current one. */
+    private isChapterBoundary;
+    /** Split a chapter's cues into runs of consecutive same-speaker cues. */
+    private groupBySpeaker;
+    /** Wrap a cue list with its computed time span. */
+    private toGroup;
+    /** Emit chapter segments, plus per-speaker children when configured. */
+    private buildSegments;
+    /** Append one child segment per speaker run within a chapter. */
+    private appendSubChapters;
+    /** Build a segment carrying both the media time-window and the transcript text. */
+    private toRawSegment;
+    /** Prefix a cue with its speaker so the transcript reads as dialogue. */
+    private formatCue;
+    /** The only speaker in the group, or undefined when several speak. */
+    private dominantSpeaker;
+    /** Infer audio vs. video from the media mime type, else use the configured default. */
+    private resolveModality;
+    /** Merge caller options with transcript-specific defaults. */
+    private resolveTranscriptOptions;
+    /** Estimated tokens for a set of cues. */
+    private textTokens;
+    /** Estimated tokens for one cue. */
+    private cueTokens;
+}
+//# sourceMappingURL=TranscriptSegmenter.d.ts.map

package/dist/generic/TranscriptSegmenter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TranscriptSegmenter.d.ts","sourceRoot":"","sources":["../../src/generic/TranscriptSegmenter.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EACH,eAAe,EACf,UAAU,EACV,mBAAmB,EACnB,kBAAkB,EAErB,MAAM,sBAAsB,CAAC;AAE9B,wDAAwD;AACxD,eAAO,MAAM,wBAAwB,eAAe,CAAC;AAErD,uDAAuD;AACvD,MAAM,WAAW,6BAA8B,SAAQ,mBAAmB;IACtE,oFAAoF;IACpF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,2FAA2F;IAC3F,eAAe,CAAC,EAAE,OAAO,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC;CACjE;AASD;;;;;;;;;;;;;;;;GAgBG;AACH,qBACa,mBAAoB,SAAQ,aAAa;IAClD,IAAW,GAAG,IAAI,MAAM,CAEvB;IAED,IAAW,mBAAmB,IAAI,eAAe,EAAE,CAElD;IAED,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,6BAA6B,CAAC,GAAG,UAAU,EAAE;IAc9F,yFAAyF;IACzF,OAAO,CAAC,QAAQ;IAIhB,mFAAmF;IACnF,OAAO,CAAC,iBAAiB;IAiBzB,sFAAsF;IACtF,OAAO,CAAC,iBAAiB;IAezB,yEAAyE;IACzE,OAAO,CAAC,cAAc;IAiBtB,mDAAmD;IACnD,OAAO,CAAC,OAAO;IAYf,wEAAwE;IACxE,OAAO,CAAC,aAAa;IAkBrB,iEAAiE;IACjE,OAAO,CAAC,iBAAiB;IAkBzB,mFAAmF;IACnF,OAAO,CAAC,YAAY;IAiBpB,yEAAyE;IACzE,OAAO,CAAC,SAAS;IAIjB,sEAAsE;IACtE,OAAO,CAAC,eAAe;IAKvB,uFAAuF;IACvF,OAAO,CAAC,eAAe;IAkBvB,8DAA8D;IAC9D,OAAO,CAAC,wBAAwB;IAUhC,0CAA0C;IAC1C,OAAO,CAAC,UAAU;IAIlB,oCAAoC;IACpC,OAAO,CAAC,SAAS;CAGpB"}

package/dist/generic/TranscriptSegmenter.js

@@ -0,0 +1,194 @@
+/**
+ * @fileoverview Transcript-driven segmenter — turns timed cues into chapters.
+ *
+ * @module @memberjunction/ai-segmentation
+ */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+import { RegisterClass } from '@memberjunction/global';
+import { BaseSegmenter } from './BaseSegmenter.js';
+/** Registration key for {@link TranscriptSegmenter}. */
+export const TRANSCRIPT_SEGMENTER_KEY = 'Transcript';
+/**
+ * Segments audio/video into **chapters** using its timed transcript.
+ *
+ * A 60-minute session recording embedded as a single vector is a mush vector —
+ * multimodal embedders only see a short window, and no single vector can represent
+ * an hour of distinct topics. This segmenter finds real boundaries first (silence
+ * gaps, speaker changes, a duration ceiling) and emits time-windowed chapters that
+ * each carry **both** a media reference (with `StartMs`/`EndMs`) *and* the
+ * transcript text for that window.
+ *
+ * That dual payload is deliberate: the media reference is what a multimodal model
+ * embeds for retrieval, while the transcript is what an agent can actually read,
+ * what a cross-encoder can rerank, and what keyword search can match.
+ *
+ * Cues come from any ASR source, including MemberJunction's realtime session
+ * capture, which already records per-turn text with speaker labels and timings.
+ */
+let TranscriptSegmenter = class TranscriptSegmenter extends BaseSegmenter {
+    get Key() {
+        return TRANSCRIPT_SEGMENTER_KEY;
+    }
+    get SupportedModalities() {
+        return ['audio', 'video', 'multimodal'];
+    }
+    SegmentCore(params) {
+        const cues = this.sortCues(params.Cues ?? []);
+        if (cues.length === 0) {
+            return [];
+        }
+        const settings = this.resolveTranscriptOptions(params.Options);
+        const chapters = this.groupIntoChapters(cues, settings);
+        return this.buildSegments(chapters, params, settings);
+    }
+    // ─────────────────────────────────────────────
+    // Grouping
+    // ─────────────────────────────────────────────
+    /** Cues must be time-ordered before boundary detection; callers may supply any order. */
+    sortCues(cues) {
+        return [...cues].filter((c) => !!c && c.Text.trim().length > 0).sort((a, b) => a.StartMs - b.StartMs);
+    }
+    /** Accumulate cues into chapters, breaking on gap, duration, or token pressure. */
+    groupIntoChapters(cues, settings) {
+        const chapters = [];
+        let current = [];
+        for (const cue of cues) {
+            if (current.length > 0 && this.isChapterBoundary(current, cue, settings)) {
+                chapters.push(this.toGroup(current));
+                current = [];
+            }
+            current.push(cue);
+        }
+        if (current.length > 0) {
+            chapters.push(this.toGroup(current));
+        }
+        return chapters;
+    }
+    /** True when `next` should start a new chapter rather than extend the current one. */
+    isChapterBoundary(current, next, settings) {
+        const previous = current[current.length - 1];
+        if (next.StartMs - previous.EndMs >= settings.BoundaryGapMs) {
+            return true;
+        }
+        if (next.EndMs - current[0].StartMs > settings.MaxChapterMs) {
+            return true;
+        }
+        return this.textTokens(current) + this.cueTokens(next) > settings.MaxSegmentTokens;
+    }
+    /** Split a chapter's cues into runs of consecutive same-speaker cues. */
+    groupBySpeaker(cues) {
+        const runs = [];
+        let current = [];
+        for (const cue of cues) {
+            const previous = current[current.length - 1];
+            if (previous && previous.Speaker !== cue.Speaker) {
+                runs.push(this.toGroup(current));
+                current = [];
+            }
+            current.push(cue);
+        }
+        if (current.length > 0) {
+            runs.push(this.toGroup(current));
+        }
+        return runs;
+    }
+    /** Wrap a cue list with its computed time span. */
+    toGroup(cues) {
+        return {
+            Cues: cues,
+            StartMs: cues[0].StartMs,
+            EndMs: cues[cues.length - 1].EndMs,
+        };
+    }
+    // ─────────────────────────────────────────────
+    // Segment construction
+    // ─────────────────────────────────────────────
+    /** Emit chapter segments, plus per-speaker children when configured. */
+    buildSegments(chapters, params, settings) {
+        const modality = this.resolveModality(params, settings);
+        const segments = [];
+        chapters.forEach((chapter, index) => {
+            segments.push(this.toRawSegment(chapter, modality, params, `Chapter ${index + 1}`));
+            const chapterIndex = segments.length - 1;
+            if (settings.EmitSubChapters) {
+                this.appendSubChapters(segments, chapter, modality, params, chapterIndex);
+            }
+        });
+        return segments;
+    }
+    /** Append one child segment per speaker run within a chapter. */
+    appendSubChapters(segments, chapter, modality, params, parentIndex) {
+        const runs = this.groupBySpeaker(chapter.Cues);
+        if (runs.length <= 1) {
+            return;
+        }
+        for (const run of runs) {
+            const child = this.toRawSegment(run, modality, params, run.Cues[0].Speaker);
+            child.ParentIndex = parentIndex;
+            segments.push(child);
+        }
+    }
+    /** Build a segment carrying both the media time-window and the transcript text. */
+    toRawSegment(group, modality, params, title) {
+        return {
+            Modality: modality,
+            Text: group.Cues.map((c) => this.formatCue(c)).join(' ').trim(),
+            Media: params.Media,
+            Title: title,
+            StartMs: group.StartMs,
+            EndMs: group.EndMs,
+            Speaker: this.dominantSpeaker(group.Cues),
+        };
+    }
+    /** Prefix a cue with its speaker so the transcript reads as dialogue. */
+    formatCue(cue) {
+        return cue.Speaker ? `${cue.Speaker}: ${cue.Text.trim()}` : cue.Text.trim();
+    }
+    /** The only speaker in the group, or undefined when several speak. */
+    dominantSpeaker(cues) {
+        const speakers = new Set(cues.map((c) => c.Speaker).filter((s) => !!s));
+        return speakers.size === 1 ? [...speakers][0] : undefined;
+    }
+    /** Infer audio vs. video from the media mime type, else use the configured default. */
+    resolveModality(params, settings) {
+        const mime = (params.Media?.MimeType ?? params.MimeType ?? '').toLowerCase();
+        if (mime.startsWith('video/')) {
+            return 'video';
+        }
+        if (mime.startsWith('audio/')) {
+            return 'audio';
+        }
+        return settings.DefaultModality;
+    }
+    // ─────────────────────────────────────────────
+    // Helpers
+    // ─────────────────────────────────────────────
+    /** Merge caller options with transcript-specific defaults. */
+    resolveTranscriptOptions(options) {
+        return {
+            ...this.resolveOptions(options),
+            MaxChapterMs: options?.MaxChapterMs ?? 300_000,
+            BoundaryGapMs: options?.BoundaryGapMs ?? 4_000,
+            EmitSubChapters: options?.EmitSubChapters ?? false,
+            DefaultModality: options?.DefaultModality ?? 'audio',
+        };
+    }
+    /** Estimated tokens for a set of cues. */
+    textTokens(cues) {
+        return cues.reduce((sum, cue) => sum + this.cueTokens(cue), 0);
+    }
+    /** Estimated tokens for one cue. */
+    cueTokens(cue) {
+        return this.tokensOf({ Modality: 'text', Text: cue.Text });
+    }
+};
+TranscriptSegmenter = __decorate([
+    RegisterClass(BaseSegmenter, TRANSCRIPT_SEGMENTER_KEY)
+], TranscriptSegmenter);
+export { TranscriptSegmenter };
+//# sourceMappingURL=TranscriptSegmenter.js.map

package/dist/generic/TranscriptSegmenter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TranscriptSegmenter.js","sourceRoot":"","sources":["../../src/generic/TranscriptSegmenter.ts"],"names":[],"mappings":"AAAA;;;;GAIG;;;;;;;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAShD,wDAAwD;AACxD,MAAM,CAAC,MAAM,wBAAwB,GAAG,YAAY,CAAC;AA4BrD;;;;;;;;;;;;;;;;GAgBG;AAEI,IAAM,mBAAmB,GAAzB,MAAM,mBAAoB,SAAQ,aAAa;IAClD,IAAW,GAAG;QACV,OAAO,wBAAwB,CAAC;IACpC,CAAC;IAED,IAAW,mBAAmB;QAC1B,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IAC5C,CAAC;IAES,WAAW,CAAC,MAAyD;QAC3E,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAC9C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO,EAAE,CAAC;QACd,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC1D,CAAC;IAED,gDAAgD;IAChD,WAAW;IACX,gDAAgD;IAEhD,yFAAyF;IACjF,QAAQ,CAAC,IAAqB;QAClC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAC1G,CAAC;IAED,mFAAmF;IAC3E,iBAAiB,CAAC,IAAqB,EAAE,QAAiD;QAC9F,MAAM,QAAQ,GAAe,EAAE,CAAC;QAChC,IAAI,OAAO,GAAoB,EAAE,CAAC;QAElC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACvE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;gBACrC,OAAO,GAAG,EAAE,CAAC;YACjB,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QACzC,CAAC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,sFAAsF;IAC9E,iBAAiB,CACrB,OAAwB,EACxB,IAAmB,EACnB,QAAiD;QAEjD,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC7C,IAAI,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC1D,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,QAAQ,CAAC,YAAY,EAAE,CAAC;YAC1D,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,gBAAgB,CAAC;IACvF,CAAC;IAED,yEAAyE;IACjE,cAAc,CAAC,IAAqB;QACxC,MAAM,IAAI,GAAe,EAAE,CAAC;QAC5B,IAAI,OAAO,GAAoB,EAAE,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC7C,IAAI,QAAQ,IAAI,QAAQ,CAAC,OAAO,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC;gBAC/C,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;gBACjC,OAAO,GAAG,EAAE,CAAC;YACjB,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QACrC,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,mDAAmD;IAC3C,OAAO,CAAC,IAAqB;QACjC,OAAO;YACH,IAAI,EAAE,IAAI;YACV,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO;YACxB,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK;SACrC,CAAC;IACN,CAAC;IAED,gDAAgD;IAChD,uBAAuB;IACvB,gDAAgD;IAEhD,wEAAwE;IAChE,aAAa,CACjB,QAAoB,EACpB,MAAyD,EACzD,QAAiD;QAEjD,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAiB,EAAE,CAAC;QAElC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YAChC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YACpF,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YACzC,IAAI,QAAQ,CAAC,eAAe,EAAE,CAAC;gBAC3B,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;YAC9E,CAAC;QACL,CAAC,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,iEAAiE;IACzD,iBAAiB,CACrB,QAAsB,EACtB,OAAiB,EACjB,QAAyB,EACzB,MAAyD,EACzD,WAAmB;QAEnB,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC/C,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC5E,KAAK,CAAC,WAAW,GAAG,WAAW,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;IACL,CAAC;IAED,mFAAmF;IAC3E,YAAY,CAChB,KAAe,EACf,QAAyB,EACzB,MAAyD,EACzD,KAAc;QAEd,OAAO;YACH,QAAQ,EAAE,QAAQ;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE;YAC/D,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,OAAO,EAAE,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC;SAC5C,CAAC;IACN,CAAC;IAED,yEAAyE;IACjE,SAAS,CAAC,GAAkB;QAChC,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAChF,CAAC;IAED,sEAAsE;IAC9D,eAAe,CAAC,IAAqB;QACzC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrF,OAAO,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9D,CAAC;IAED,uFAAuF;IAC/E,eAAe,CACnB,MAAyD,EACzD,QAAiD;QAEjD,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAC7E,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5B,OAAO,OAAO,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5B,OAAO,OAAO,CAAC;QACnB,CAAC;QACD,OAAO,QAAQ,CAAC,eAAe,CAAC;IACpC,CAAC;IAED,gDAAgD;IAChD,UAAU;IACV,gDAAgD;IAEhD,8DAA8D;IACtD,wBAAwB,CAAC,OAAuC;QACpE,OAAO;YACH,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;YAC/B,YAAY,EAAE,OAAO,EAAE,YAAY,IAAI,OAAO;YAC9C,aAAa,EAAE,OAAO,EAAE,aAAa,IAAI,KAAK;YAC9C,eAAe,EAAE,OAAO,EAAE,eAAe,IAAI,KAAK;YAClD,eAAe,EAAE,OAAO,EAAE,eAAe,IAAI,OAAO;SACvD,CAAC;IACN,CAAC;IAED,0CAA0C;IAClC,UAAU,CAAC,IAAqB;QACpC,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACnE,CAAC;IAED,oCAAoC;IAC5B,SAAS,CAAC,GAAkB;QAChC,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;CACJ,CAAA;AAvMY,mBAAmB;IAD/B,aAAa,CAAC,aAAa,EAAE,wBAAwB,CAAC;GAC1C,mBAAmB,CAuM/B"}

package/dist/index.d.ts

@@ -0,0 +1,33 @@
+/**
+ * @memberjunction/ai-segmentation
+ *
+ * Pluggable content segmentation for RAG ingestion. Importing this package
+ * registers every built-in segmenter with the MJ class factory, so
+ * `BaseSegmenter.Resolve(key)` works for any of them.
+ */
+export * from './generic/Segmentation.types.js';
+export * from './generic/BaseSegmenter.js';
+export * from './generic/StructuralTextSegmenter.js';
+export * from './generic/SemanticTextSegmenter.js';
+export * from './generic/TranscriptSegmenter.js';
+export * from './generic/FixedWindowSegmenter.js';
+export * from './generic/AdaptiveBoundarySegmenter.js';
+export * from './generic/PagedContentSegmenter.js';
+export * from './generic/BaseContentCleaner.js';
+export * from './generic/HtmlContentCleaner.js';
+export * from './generic/PlainTextContentCleaner.js';
+export * from './generic/SegmentationResolver.js';
+/**
+ * Load-prevention export.
+ *
+ * Modern bundlers (ESBuild, Vite) tree-shake classes that are only ever instantiated
+ * dynamically through MJ's `ClassFactory` — which is exactly how segmenters and cleaners
+ * are resolved. If a built-in is dropped from the bundle, `Resolve()` returns null and the
+ * resolver silently degrades to its fallback: content still gets cleaned and chunked, just
+ * by the wrong strategy, with no error to show for it.
+ *
+ * Call this no-op from the consuming application's bootstrap to force a static reference
+ * so the `@RegisterClass`-decorated segmenters are retained.
+ */
+export declare function LoadContentSegmenters(): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,cAAc,8BAA8B,CAAC;AAC7C,cAAc,yBAAyB,CAAC;AACxC,cAAc,mCAAmC,CAAC;AAClD,cAAc,iCAAiC,CAAC;AAChD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,mCAAmC,CAAC;AAClD,cAAc,gCAAgC,CAAC;AAE/C;;;;;;;;;;;GAWG;AACH,wBAAgB,qBAAqB,IAAI,IAAI,CAG5C"}

package/dist/index.js

@@ -0,0 +1,36 @@
+/**
+ * @memberjunction/ai-segmentation
+ *
+ * Pluggable content segmentation for RAG ingestion. Importing this package
+ * registers every built-in segmenter with the MJ class factory, so
+ * `BaseSegmenter.Resolve(key)` works for any of them.
+ */
+export * from './generic/Segmentation.types.js';
+export * from './generic/BaseSegmenter.js';
+export * from './generic/StructuralTextSegmenter.js';
+export * from './generic/SemanticTextSegmenter.js';
+export * from './generic/TranscriptSegmenter.js';
+export * from './generic/FixedWindowSegmenter.js';
+export * from './generic/AdaptiveBoundarySegmenter.js';
+export * from './generic/PagedContentSegmenter.js';
+export * from './generic/BaseContentCleaner.js';
+export * from './generic/HtmlContentCleaner.js';
+export * from './generic/PlainTextContentCleaner.js';
+export * from './generic/SegmentationResolver.js';
+/**
+ * Load-prevention export.
+ *
+ * Modern bundlers (ESBuild, Vite) tree-shake classes that are only ever instantiated
+ * dynamically through MJ's `ClassFactory` — which is exactly how segmenters and cleaners
+ * are resolved. If a built-in is dropped from the bundle, `Resolve()` returns null and the
+ * resolver silently degrades to its fallback: content still gets cleaned and chunked, just
+ * by the wrong strategy, with no error to show for it.
+ *
+ * Call this no-op from the consuming application's bootstrap to force a static reference
+ * so the `@RegisterClass`-decorated segmenters are retained.
+ */
+export function LoadContentSegmenters() {
+    // Intentionally empty — importing this module is what fires the @RegisterClass
+    // decorators; this function exists purely as a static import target.
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,cAAc,8BAA8B,CAAC;AAC7C,cAAc,yBAAyB,CAAC;AACxC,cAAc,mCAAmC,CAAC;AAClD,cAAc,iCAAiC,CAAC;AAChD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,mCAAmC,CAAC;AAClD,cAAc,gCAAgC,CAAC;AAE/C;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,qBAAqB;IACjC,+EAA+E;IAC/E,qEAAqE;AACzE,CAAC"}