@memberjunction/ai-segmentation 0.0.0 → 5.51.0

package/dist/generic/Segmentation.types.d.ts

@@ -0,0 +1,198 @@
+/**
+ * @fileoverview Type contract for content segmentation.
+ *
+ * Segmentation is the step that turns a piece of source content (a document, a
+ * recording, an image) into an ordered list of embeddable {@link ContentSegment}s.
+ * It sits *upstream* of embedding: a segmenter decides **what** gets embedded,
+ * the embedding model decides **how**.
+ *
+ * This is deliberately separate from `TextChunker` (in `@memberjunction/ai-vectors`). `TextChunker` answers
+ * "how do I split this string so it fits a token budget"; a segmenter answers
+ * "what are the meaningful units of this content" — which may be sections of a
+ * document, chapters of a recording, or a single image. Segmenters typically
+ * *use* `TextChunker` to enforce the token budget within a unit they identified.
+ *
+ * @module @memberjunction/ai-segmentation
+ */
+import { UserInfo } from '@memberjunction/core';
+/**
+ * The modality of a segment's payload. Drives downstream index routing
+ * (text vs. multimodal vector index) and retrieval-time fusion.
+ */
+export type ContentModality = 'text' | 'image' | 'audio' | 'video' | 'multimodal';
+/**
+ * A pointer to non-text content. Segmenters emit these for media segments
+ * instead of (or alongside) `Text`.
+ *
+ * Exactly one of `URL`, `Base64Data`, or (`StorageProviderKey` + `ObjectKey`)
+ * is expected to be populated; consumers resolve whichever is present.
+ */
+export interface MediaReference {
+    /** Directly fetchable URL (http(s) or data URL). */
+    URL?: string;
+    /** `@memberjunction/storage` provider key, when the media lives in MJ file storage. */
+    StorageProviderKey?: string;
+    /** Object/blob key within the storage provider. */
+    ObjectKey?: string;
+    /** IANA mime type, e.g. `video/mp4`. Used to gate provider capability checks. */
+    MimeType?: string;
+    /** Inline base64 payload (no data-URL prefix). Prefer URL/storage refs for large media. */
+    Base64Data?: string;
+}
+/**
+ * A single timed transcript cue — the unit produced by ASR and by MJ's
+ * realtime session capture (which records speaker + timings per turn).
+ */
+export interface TranscriptCue {
+    /** Cue start, milliseconds from the beginning of the asset. */
+    StartMs: number;
+    /** Cue end, milliseconds from the beginning of the asset. */
+    EndMs: number;
+    /** Spoken text for this cue. */
+    Text: string;
+    /** Optional speaker label/id — a speaker change is a strong boundary signal. */
+    Speaker?: string;
+}
+/**
+ * One embeddable unit of content produced by a segmenter.
+ *
+ * A segment carries `Text`, `Media`, or **both** (the "dual representation" case:
+ * a video chapter with a native media reference *and* its transcript, so it can be
+ * embedded natively for retrieval while remaining readable for an agent).
+ */
+export interface ContentSegment {
+    /** 0-based position within the full segment list. Assigned by {@link BaseSegmenter}. */
+    Sequence: number;
+    /** Payload modality. */
+    Modality: ContentModality;
+    /** Textual payload — the extracted/transcribed text for this segment. */
+    Text?: string;
+    /** Media payload pointer, for non-text segments. */
+    Media?: MediaReference;
+    /** Human-readable label, e.g. a heading or a generated chapter title. */
+    Title?: string;
+    /** Inclusive start character offset within the source text. */
+    StartOffset?: number;
+    /** Exclusive end character offset within the source text. */
+    EndOffset?: number;
+    /** Segment start in milliseconds, for audio/video. */
+    StartMs?: number;
+    /** Segment end in milliseconds, for audio/video. */
+    EndMs?: number;
+    /** 1-based page number, for paginated sources (PDF, slides). */
+    PageNumber?: number;
+    /** `Sequence` of this segment's parent, for chapter -> sub-chapter hierarchies. */
+    ParentSequence?: number;
+    /** Nesting depth; 0 for top-level segments. */
+    Depth: number;
+    /** Estimated token count of `Text` (0 for pure-media segments). */
+    TokenEstimate: number;
+    /** Registration key of the segmenter that produced this segment — provenance. */
+    SegmenterKey: string;
+    /** Speaker label carried through from transcript cues, when known. */
+    Speaker?: string;
+}
+/**
+ * A segment as emitted by a concrete segmenter's `SegmentCore`, before the base
+ * class normalizes it (assigns `Sequence`/`Depth`/`TokenEstimate`, enforces the
+ * token ceiling, and resolves parent links).
+ *
+ * `ParentIndex` refers to the **index within the raw array** returned by
+ * `SegmentCore`; the base class remaps it to a real `Sequence` afterwards, which
+ * keeps subclasses from having to reason about post-split numbering.
+ */
+export interface RawSegment {
+    Modality: ContentModality;
+    Text?: string;
+    Media?: MediaReference;
+    Title?: string;
+    StartOffset?: number;
+    EndOffset?: number;
+    StartMs?: number;
+    EndMs?: number;
+    PageNumber?: number;
+    Speaker?: string;
+    /** Index into the raw segment array identifying this segment's parent. */
+    ParentIndex?: number;
+}
+/**
+ * One page of a paginated source (PDF, slide deck).
+ *
+ * A page may carry extracted text, a media reference to the rendered page, or both —
+ * the both case is what lets a page be embedded natively by a multimodal model (preserving
+ * tables and charts that text extraction flattens) while its text remains available for
+ * lexical search.
+ */
+export interface ContentPage {
+    /** One-based page number. */
+    PageNumber: number;
+    /** Extracted text for this page. */
+    Text?: string;
+    /** Reference to the rendered page image, when available. */
+    Media?: MediaReference;
+}
+/**
+ * Common knobs understood by every segmenter. Concrete segmenters extend this
+ * with their own strongly-typed options rather than accepting a loose bag.
+ */
+export interface SegmentationOptions {
+    /**
+     * Hard ceiling on tokens per text segment. The base class splits any
+     * oversized segment via `TextChunker` so no segmenter can exceed it.
+     * Default: 512.
+     */
+    MaxSegmentTokens?: number;
+    /** Overlap tokens applied when an oversized segment must be split. Default: 10% of max. */
+    OverlapTokens?: number;
+    /**
+     * Segments whose text estimates below this many tokens are merged forward into
+     * the next segment, preventing a spray of near-empty vectors. Default: 0 (off).
+     */
+    MinSegmentTokens?: number;
+}
+/**
+ * Input to {@link BaseSegmenter.Segment}.
+ *
+ * @typeParam TOptions - the concrete segmenter's options type.
+ */
+export interface SegmentationParams<TOptions extends SegmentationOptions = SegmentationOptions> {
+    /** Extracted text of the source content, when it has any. */
+    Text?: string;
+    /** Media pointer for the source asset, for image/audio/video content. */
+    Media?: MediaReference;
+    /** Timed transcript cues, when available (ASR output or MJ realtime capture). */
+    Cues?: TranscriptCue[];
+    /** Pages of a paginated source, for page-aware segmentation. */
+    Pages?: ContentPage[];
+    /** Total duration of the source asset in milliseconds, for AV content. */
+    DurationMs?: number;
+    /** Mime type of the source asset — lets a segmenter pick a structure parser. */
+    MimeType?: string;
+    /**
+     * Context user, required by segmenters that call MJ services (e.g. the LLM
+     * boundary pass in `SemanticTextSegmenter`). Always pass it in server-side code.
+     */
+    ContextUser?: UserInfo;
+    /** Strategy-specific options. */
+    Options?: TOptions;
+}
+/**
+ * Result of a segmentation pass. Segmenters never throw for content-shaped
+ * problems — they return `Success: false` with an `ErrorMessage`, matching the
+ * convention used by `RunView` and `BaseEntity.Save`.
+ */
+export interface SegmentationResult {
+    /** False when segmentation could not be performed. */
+    Success: boolean;
+    /** The produced segments, in order. Empty when `Success` is false. */
+    Segments: ContentSegment[];
+    /** Registration key of the segmenter that ran. */
+    SegmenterKey: string;
+    /** Populated when `Success` is false. */
+    ErrorMessage?: string;
+    /** Non-fatal notes — e.g. "no cues supplied, fell back to fixed windows". */
+    Warnings: string[];
+}
+/** Default token ceiling applied when a caller does not specify one. */
+export declare const DEFAULT_MAX_SEGMENT_TOKENS = 512;
+//# sourceMappingURL=Segmentation.types.d.ts.map

package/dist/generic/Segmentation.types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Segmentation.types.d.ts","sourceRoot":"","sources":["../../src/generic/Segmentation.types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAEhD;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,YAAY,CAAC;AAElF;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC3B,oDAAoD;IACpD,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,uFAAuF;IACvF,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iFAAiF;IACjF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2FAA2F;IAC3F,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC1B,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,6DAA6D;IAC7D,KAAK,EAAE,MAAM,CAAC;IACd,gCAAgC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,gFAAgF;IAChF,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC3B,wFAAwF;IACxF,QAAQ,EAAE,MAAM,CAAC;IACjB,wBAAwB;IACxB,QAAQ,EAAE,eAAe,CAAC;IAC1B,yEAAyE;IACzE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oDAAoD;IACpD,KAAK,CAAC,EAAE,cAAc,CAAC;IACvB,yEAAyE;IACzE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sDAAsD;IACtD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,oDAAoD;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mFAAmF;IACnF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd,mEAAmE;IACnE,aAAa,EAAE,MAAM,CAAC;IACtB,iFAAiF;IACjF,YAAY,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,UAAU;IACvB,QAAQ,EAAE,eAAe,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,cAAc,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0EAA0E;IAC1E,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,WAAW;IACxB,6BAA6B;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,oCAAoC;IACpC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,4DAA4D;IAC5D,KAAK,CAAC,EAAE,cAAc,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAChC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,2FAA2F;IAC3F,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;;GAIG;AACH,MAAM,WAAW,kBAAkB,CAAC,QAAQ,SAAS,mBAAmB,GAAG,mBAAmB;IAC1F,6DAA6D;IAC7D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,yEAAyE;IACzE,KAAK,CAAC,EAAE,cAAc,CAAC;IACvB,iFAAiF;IACjF,IAAI,CAAC,EAAE,aAAa,EAAE,CAAC;IACvB,gEAAgE;IAChE,KAAK,CAAC,EAAE,WAAW,EAAE,CAAC;IACtB,0EAA0E;IAC1E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gFAAgF;IAChF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,iCAAiC;IACjC,OAAO,CAAC,EAAE,QAAQ,CAAC;CACtB;AAED;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IAC/B,sDAAsD;IACtD,OAAO,EAAE,OAAO,CAAC;IACjB,sEAAsE;IACtE,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,kDAAkD;IAClD,YAAY,EAAE,MAAM,CAAC;IACrB,yCAAyC;IACzC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,6EAA6E;IAC7E,QAAQ,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,wEAAwE;AACxE,eAAO,MAAM,0BAA0B,MAAM,CAAC"}

package/dist/generic/Segmentation.types.js

@@ -0,0 +1,19 @@
+/**
+ * @fileoverview Type contract for content segmentation.
+ *
+ * Segmentation is the step that turns a piece of source content (a document, a
+ * recording, an image) into an ordered list of embeddable {@link ContentSegment}s.
+ * It sits *upstream* of embedding: a segmenter decides **what** gets embedded,
+ * the embedding model decides **how**.
+ *
+ * This is deliberately separate from `TextChunker` (in `@memberjunction/ai-vectors`). `TextChunker` answers
+ * "how do I split this string so it fits a token budget"; a segmenter answers
+ * "what are the meaningful units of this content" — which may be sections of a
+ * document, chapters of a recording, or a single image. Segmenters typically
+ * *use* `TextChunker` to enforce the token budget within a unit they identified.
+ *
+ * @module @memberjunction/ai-segmentation
+ */
+/** Default token ceiling applied when a caller does not specify one. */
+export const DEFAULT_MAX_SEGMENT_TOKENS = 512;
+//# sourceMappingURL=Segmentation.types.js.map

package/dist/generic/Segmentation.types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Segmentation.types.js","sourceRoot":"","sources":["../../src/generic/Segmentation.types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AA+LH,wEAAwE;AACxE,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC"}

package/dist/generic/SegmentationResolver.d.ts

@@ -0,0 +1,43 @@
+/**
+ * @fileoverview Helpers for selecting a segmenter from configuration or content shape.
+ *
+ * @module @memberjunction/ai-segmentation
+ */
+import { BaseSegmenter } from './BaseSegmenter.js';
+import { BaseContentCleaner } from './BaseContentCleaner.js';
+import { SegmentationParams } from './Segmentation.types.js';
+/**
+ * Resolve a segmenter by registration key, falling back safely.
+ *
+ * Configuration is data, and data drifts — a `Content Type` may name a segmenter
+ * that has been renamed or lives in a package this process didn't load. Rather
+ * than throw mid-ingestion, an unresolvable key logs and degrades to the
+ * fixed-window segmenter, which can segment anything.
+ *
+ * @param key - registration key from metadata; when omitted the fallback is used.
+ * @param fallbackKey - key to try before the built-in last resort.
+ */
+export declare function ResolveSegmenter(key?: string, fallbackKey?: string): BaseSegmenter;
+/**
+ * Suggest the best-fit segmenter key for a piece of content.
+ *
+ * The ordering encodes the quality hierarchy: a real transcript beats document
+ * structure, which beats uniform windows. Callers should treat this as a default
+ * that explicit configuration may override.
+ */
+export declare function SuggestSegmenterKey(params: SegmentationParams): string;
+/**
+ * Resolve a content cleaner by registration key, falling back safely.
+ *
+ * Mirrors {@link ResolveSegmenter}: an unresolvable key logs and degrades to the
+ * plain-text cleaner (whitespace normalization only) rather than throwing mid-ingestion.
+ */
+export declare function ResolveContentCleaner(key?: string, fallbackKey?: string): BaseContentCleaner;
+/**
+ * Suggest a cleaner for a piece of content based on its mime type.
+ *
+ * HTML is the only format that genuinely needs structural cleaning; everything else is
+ * already text and only wants whitespace normalization.
+ */
+export declare function SuggestCleanerKey(mimeType?: string): string;
+//# sourceMappingURL=SegmentationResolver.d.ts.map

package/dist/generic/SegmentationResolver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SegmentationResolver.d.ts","sourceRoot":"","sources":["../../src/generic/SegmentationResolver.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAKhD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAG1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,aAAa,CAUlF;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,kBAAkB,GAAG,MAAM,CAWtE;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,kBAAkB,CAU5F;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAG3D"}

package/dist/generic/SegmentationResolver.js

@@ -0,0 +1,83 @@
+/**
+ * @fileoverview Helpers for selecting a segmenter from configuration or content shape.
+ *
+ * @module @memberjunction/ai-segmentation
+ */
+import { LogStatus } from '@memberjunction/core';
+import { BaseSegmenter } from './BaseSegmenter.js';
+import { FIXED_WINDOW_SEGMENTER_KEY, FixedWindowSegmenter } from './FixedWindowSegmenter.js';
+import { STRUCTURAL_TEXT_SEGMENTER_KEY } from './StructuralTextSegmenter.js';
+import { TRANSCRIPT_SEGMENTER_KEY } from './TranscriptSegmenter.js';
+import { PAGED_CONTENT_SEGMENTER_KEY } from './PagedContentSegmenter.js';
+import { BaseContentCleaner } from './BaseContentCleaner.js';
+import { HTML_CONTENT_CLEANER_KEY } from './HtmlContentCleaner.js';
+import { PLAIN_TEXT_CONTENT_CLEANER_KEY, PlainTextContentCleaner } from './PlainTextContentCleaner.js';
+/**
+ * Resolve a segmenter by registration key, falling back safely.
+ *
+ * Configuration is data, and data drifts — a `Content Type` may name a segmenter
+ * that has been renamed or lives in a package this process didn't load. Rather
+ * than throw mid-ingestion, an unresolvable key logs and degrades to the
+ * fixed-window segmenter, which can segment anything.
+ *
+ * @param key - registration key from metadata; when omitted the fallback is used.
+ * @param fallbackKey - key to try before the built-in last resort.
+ */
+export function ResolveSegmenter(key, fallbackKey) {
+    const requested = key ? BaseSegmenter.Resolve(key) : null;
+    if (requested) {
+        return requested;
+    }
+    if (key) {
+        LogStatus(`[Segmentation] Segmenter '${key}' is not registered — falling back.`);
+    }
+    const fallback = fallbackKey ? BaseSegmenter.Resolve(fallbackKey) : null;
+    return fallback ?? BaseSegmenter.Resolve(FIXED_WINDOW_SEGMENTER_KEY) ?? new FixedWindowSegmenter();
+}
+/**
+ * Suggest the best-fit segmenter key for a piece of content.
+ *
+ * The ordering encodes the quality hierarchy: a real transcript beats document
+ * structure, which beats uniform windows. Callers should treat this as a default
+ * that explicit configuration may override.
+ */
+export function SuggestSegmenterKey(params) {
+    if (params.Cues && params.Cues.length > 0) {
+        return TRANSCRIPT_SEGMENTER_KEY;
+    }
+    if (params.Pages && params.Pages.length > 0) {
+        return PAGED_CONTENT_SEGMENTER_KEY;
+    }
+    if (params.Text && params.Text.trim().length > 0) {
+        return STRUCTURAL_TEXT_SEGMENTER_KEY;
+    }
+    return FIXED_WINDOW_SEGMENTER_KEY;
+}
+/**
+ * Resolve a content cleaner by registration key, falling back safely.
+ *
+ * Mirrors {@link ResolveSegmenter}: an unresolvable key logs and degrades to the
+ * plain-text cleaner (whitespace normalization only) rather than throwing mid-ingestion.
+ */
+export function ResolveContentCleaner(key, fallbackKey) {
+    const requested = key ? BaseContentCleaner.Resolve(key) : null;
+    if (requested) {
+        return requested;
+    }
+    if (key) {
+        LogStatus(`[Segmentation] Content cleaner '${key}' is not registered — falling back.`);
+    }
+    const fallback = fallbackKey ? BaseContentCleaner.Resolve(fallbackKey) : null;
+    return fallback ?? BaseContentCleaner.Resolve(PLAIN_TEXT_CONTENT_CLEANER_KEY) ?? new PlainTextContentCleaner();
+}
+/**
+ * Suggest a cleaner for a piece of content based on its mime type.
+ *
+ * HTML is the only format that genuinely needs structural cleaning; everything else is
+ * already text and only wants whitespace normalization.
+ */
+export function SuggestCleanerKey(mimeType) {
+    const mime = (mimeType ?? '').toLowerCase();
+    return mime.includes('html') || mime.includes('xml') ? HTML_CONTENT_CLEANER_KEY : PLAIN_TEXT_CONTENT_CLEANER_KEY;
+}
+//# sourceMappingURL=SegmentationResolver.js.map

package/dist/generic/SegmentationResolver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SegmentationResolver.js","sourceRoot":"","sources":["../../src/generic/SegmentationResolver.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC1F,OAAO,EAAE,6BAA6B,EAAE,MAAM,2BAA2B,CAAC;AAC1E,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,2BAA2B,EAAE,MAAM,yBAAyB,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,8BAA8B,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGpG;;;;;;;;;;GAUG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAY,EAAE,WAAoB;IAC/D,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,IAAI,SAAS,EAAE,CAAC;QACZ,OAAO,SAAS,CAAC;IACrB,CAAC;IACD,IAAI,GAAG,EAAE,CAAC;QACN,SAAS,CAAC,6BAA6B,GAAG,qCAAqC,CAAC,CAAC;IACrF,CAAC;IACD,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACzE,OAAO,QAAQ,IAAI,aAAa,CAAC,OAAO,CAAC,0BAA0B,CAAC,IAAI,IAAI,oBAAoB,EAAE,CAAC;AACvG,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAA0B;IAC1D,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxC,OAAO,wBAAwB,CAAC;IACpC,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1C,OAAO,2BAA2B,CAAC;IACvC,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/C,OAAO,6BAA6B,CAAC;IACzC,CAAC;IACD,OAAO,0BAA0B,CAAC;AACtC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAY,EAAE,WAAoB;IACpE,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/D,IAAI,SAAS,EAAE,CAAC;QACZ,OAAO,SAAS,CAAC;IACrB,CAAC;IACD,IAAI,GAAG,EAAE,CAAC;QACN,SAAS,CAAC,mCAAmC,GAAG,qCAAqC,CAAC,CAAC;IAC3F,CAAC;IACD,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9E,OAAO,QAAQ,IAAI,kBAAkB,CAAC,OAAO,CAAC,8BAA8B,CAAC,IAAI,IAAI,uBAAuB,EAAE,CAAC;AACnH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAiB;IAC/C,MAAM,IAAI,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5C,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,8BAA8B,CAAC;AACrH,CAAC"}

package/dist/generic/SemanticTextSegmenter.d.ts

@@ -0,0 +1,80 @@
+/**
+ * @fileoverview LLM-driven semantic segmenter — finds topic boundaries in prose.
+ *
+ * @module @memberjunction/ai-segmentation
+ */
+import { BaseSegmenter } from './BaseSegmenter.js';
+import { ContentModality, RawSegment, SegmentationOptions, SegmentationParams } from './Segmentation.types.js';
+/** Registration key for {@link SemanticTextSegmenter}. */
+export declare const SEMANTIC_TEXT_SEGMENTER_KEY = "SemanticText";
+/** Default name of the `MJ: AI Prompts` record driving boundary detection. */
+export declare const SEMANTIC_SEGMENTATION_PROMPT_NAME = "Content Semantic Segmentation";
+/** Options specific to {@link SemanticTextSegmenter}. */
+export interface SemanticTextSegmentationOptions extends SegmentationOptions {
+    /** Name of the `MJ: AI Prompts` record to run. Default: {@link SEMANTIC_SEGMENTATION_PROMPT_NAME}. */
+    PromptName?: string;
+    /** Optional model override (an `MJ: AI Models` ID) for the boundary pass. */
+    ModelID?: string;
+    /**
+     * Skip the LLM call entirely when the document estimates below this many
+     * tokens — short documents rarely contain multiple topics and the call would
+     * not repay its cost. Default: 750.
+     */
+    MinTokensForLLM?: number;
+    /**
+     * Maximum characters of each block shown to the model. Boundary detection only
+     * needs the opening of a block, so truncating keeps the prompt cheap on long
+     * documents. Default: 240.
+     */
+    BlockPreviewChars?: number;
+    /** Maximum blocks sent in one pass. Default: 300. */
+    MaxBlocks?: number;
+}
+/**
+ * Segments prose by asking an LLM where the topics change.
+ *
+ * Structure-aware segmentation only works when the author left structure behind.
+ * Transcripts, scanned reports, and long-form articles frequently have none — the
+ * topic shifts, but no heading marks it. This segmenter finds those latent
+ * boundaries and names them, producing titled sections that behave like headings
+ * the document never had.
+ *
+ * ## Cost posture
+ *
+ * The LLM pass is the expensive part of ingestion, so this class is written to
+ * avoid it whenever it wouldn't pay off: short documents short-circuit to
+ * structural segmentation, blocks are truncated to a preview before being shown to
+ * the model, and any failure degrades to `StructuralText` rather than failing the
+ * ingestion run. The model is asked to classify *block indices*, never character
+ * offsets — models are unreliable at arithmetic over long strings, and a wrong
+ * offset would silently corrupt chunk provenance.
+ *
+ * Because it runs through `AIPromptRunner`, every pass is a tracked `MJ: AI Prompt
+ * Run` with full token and cost attribution, and the prompt itself is versioned
+ * metadata rather than a string literal in code.
+ */
+export declare class SemanticTextSegmenter extends BaseSegmenter {
+    get Key(): string;
+    get SupportedModalities(): ContentModality[];
+    protected SegmentCore(params: SegmentationParams<SemanticTextSegmentationOptions>): Promise<RawSegment[]>;
+    /** Split the document into paragraph blocks with real offsets. */
+    private buildBlocks;
+    /** Append a block when the slice has content. */
+    private pushBlock;
+    /** Run the segmentation prompt and return validated boundary block indices. */
+    private proposeBoundaries;
+    /** Render numbered, truncated blocks for the prompt. */
+    private renderBlocks;
+    /** Accept either a parsed object or a JSON string from the runner. */
+    private parseResult;
+    /**
+     * Clamp, dedupe, and sort model output. A hallucinated or out-of-range block
+     * index must never become a chunk offset, so anything unusable is dropped here.
+     */
+    private sanitizeBoundaries;
+    /** Join blocks between consecutive boundaries into one segment each. */
+    private buildSegments;
+    /** Degrade to structural segmentation — never fail an ingestion run over segmentation. */
+    private fallback;
+}
+//# sourceMappingURL=SemanticTextSegmenter.d.ts.map

package/dist/generic/SemanticTextSegmenter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SemanticTextSegmenter.d.ts","sourceRoot":"","sources":["../../src/generic/SemanticTextSegmenter.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAOH,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE5G,0DAA0D;AAC1D,eAAO,MAAM,2BAA2B,iBAAiB,CAAC;AAE1D,8EAA8E;AAC9E,eAAO,MAAM,iCAAiC,kCAAkC,CAAC;AAEjF,yDAAyD;AACzD,MAAM,WAAW,+BAAgC,SAAQ,mBAAmB;IACxE,sGAAsG;IACtG,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6EAA6E;IAC7E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,qDAAqD;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAeD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,qBACa,qBAAsB,SAAQ,aAAa;IACpD,IAAW,GAAG,IAAI,MAAM,CAEvB;IAED,IAAW,mBAAmB,IAAI,eAAe,EAAE,CAElD;cAEe,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,+BAA+B,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IA0B/G,kEAAkE;IAClE,OAAO,CAAC,WAAW;IAcnB,iDAAiD;IACjD,OAAO,CAAC,SAAS;IAWjB,+EAA+E;YACjE,iBAAiB;IA6B/B,wDAAwD;IACxD,OAAO,CAAC,YAAY;IAKpB,sEAAsE;IACtE,OAAO,CAAC,WAAW;IAenB;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IA2B1B,wEAAwE;IACxE,OAAO,CAAC,aAAa;IAerB,0FAA0F;YAC5E,QAAQ;CAWzB"}

package/dist/generic/SemanticTextSegmenter.js

@@ -0,0 +1,201 @@
+/**
+ * @fileoverview LLM-driven semantic segmenter — finds topic boundaries in prose.
+ *
+ * @module @memberjunction/ai-segmentation
+ */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+import { LogError, LogStatus } from '@memberjunction/core';
+import { RegisterClass } from '@memberjunction/global';
+import { AIEngine } from '@memberjunction/aiengine';
+import { AIPromptRunner } from '@memberjunction/ai-prompts';
+import { AIPromptParams } from '@memberjunction/ai-core-plus';
+import { BaseSegmenter } from './BaseSegmenter.js';
+import { StructuralTextSegmenter } from './StructuralTextSegmenter.js';
+/** Registration key for {@link SemanticTextSegmenter}. */
+export const SEMANTIC_TEXT_SEGMENTER_KEY = 'SemanticText';
+/** Default name of the `MJ: AI Prompts` record driving boundary detection. */
+export const SEMANTIC_SEGMENTATION_PROMPT_NAME = 'Content Semantic Segmentation';
+/**
+ * Segments prose by asking an LLM where the topics change.
+ *
+ * Structure-aware segmentation only works when the author left structure behind.
+ * Transcripts, scanned reports, and long-form articles frequently have none — the
+ * topic shifts, but no heading marks it. This segmenter finds those latent
+ * boundaries and names them, producing titled sections that behave like headings
+ * the document never had.
+ *
+ * ## Cost posture
+ *
+ * The LLM pass is the expensive part of ingestion, so this class is written to
+ * avoid it whenever it wouldn't pay off: short documents short-circuit to
+ * structural segmentation, blocks are truncated to a preview before being shown to
+ * the model, and any failure degrades to `StructuralText` rather than failing the
+ * ingestion run. The model is asked to classify *block indices*, never character
+ * offsets — models are unreliable at arithmetic over long strings, and a wrong
+ * offset would silently corrupt chunk provenance.
+ *
+ * Because it runs through `AIPromptRunner`, every pass is a tracked `MJ: AI Prompt
+ * Run` with full token and cost attribution, and the prompt itself is versioned
+ * metadata rather than a string literal in code.
+ */
+let SemanticTextSegmenter = class SemanticTextSegmenter extends BaseSegmenter {
+    get Key() {
+        return SEMANTIC_TEXT_SEGMENTER_KEY;
+    }
+    get SupportedModalities() {
+        return ['text'];
+    }
+    async SegmentCore(params) {
+        const text = params.Text ?? '';
+        if (text.trim().length === 0) {
+            return [];
+        }
+        const minTokens = params.Options?.MinTokensForLLM ?? 750;
+        if (this.tokensOf({ Modality: 'text', Text: text }) < minTokens) {
+            return this.fallback(params);
+        }
+        const blocks = this.buildBlocks(text, params.Options);
+        if (blocks.length < 2) {
+            return this.fallback(params);
+        }
+        const boundaries = await this.proposeBoundaries(blocks, params);
+        if (!boundaries || boundaries.length === 0) {
+            return this.fallback(params);
+        }
+        return this.buildSegments(blocks, boundaries);
+    }
+    // ─────────────────────────────────────────────
+    // Block preparation
+    // ─────────────────────────────────────────────
+    /** Split the document into paragraph blocks with real offsets. */
+    buildBlocks(text, options) {
+        const blocks = [];
+        const regex = /\n\s*\n/g;
+        let cursor = 0;
+        let match = regex.exec(text);
+        while (match !== null) {
+            this.pushBlock(blocks, text, cursor, match.index);
+            cursor = match.index + match[0].length;
+            match = regex.exec(text);
+        }
+        this.pushBlock(blocks, text, cursor, text.length);
+        return blocks.slice(0, options?.MaxBlocks ?? 300);
+    }
+    /** Append a block when the slice has content. */
+    pushBlock(blocks, text, start, end) {
+        const body = text.slice(start, end).trim();
+        if (body.length > 0) {
+            blocks.push({ Index: blocks.length, Text: body, StartOffset: start, EndOffset: end });
+        }
+    }
+    // ─────────────────────────────────────────────
+    // LLM boundary detection
+    // ─────────────────────────────────────────────
+    /** Run the segmentation prompt and return validated boundary block indices. */
+    async proposeBoundaries(blocks, params) {
+        const promptName = params.Options?.PromptName ?? SEMANTIC_SEGMENTATION_PROMPT_NAME;
+        const prompt = AIEngine.Instance.Prompts.find((p) => p.Name === promptName && p.Status === 'Active');
+        if (!prompt) {
+            LogStatus(`[SemanticTextSegmenter] Prompt '${promptName}' not found — falling back to structural segmentation.`);
+            return null;
+        }
+        const promptParams = new AIPromptParams();
+        promptParams.prompt = prompt;
+        promptParams.contextUser = params.ContextUser;
+        promptParams.data = { document: this.renderBlocks(blocks, params.Options), blockCount: blocks.length };
+        promptParams.attemptJSONRepair = true;
+        promptParams.additionalParameters = { temperature: 0.0 };
+        if (params.Options?.ModelID) {
+            promptParams.override = { modelId: params.Options.ModelID };
+        }
+        const result = await new AIPromptRunner().ExecutePrompt(promptParams);
+        if (!result.success) {
+            LogError(`[SemanticTextSegmenter] Boundary prompt failed: ${result.errorMessage ?? 'unknown error'}`);
+            return null;
+        }
+        return this.sanitizeBoundaries(this.parseResult(result.result), blocks.length);
+    }
+    /** Render numbered, truncated blocks for the prompt. */
+    renderBlocks(blocks, options) {
+        const previewChars = options?.BlockPreviewChars ?? 240;
+        return blocks.map((b) => `[${b.Index}] ${b.Text.slice(0, previewChars)}`).join('\n\n');
+    }
+    /** Accept either a parsed object or a JSON string from the runner. */
+    parseResult(raw) {
+        if (!raw) {
+            return null;
+        }
+        if (typeof raw !== 'string') {
+            return raw;
+        }
+        try {
+            return JSON.parse(raw);
+        }
+        catch {
+            LogError(`[SemanticTextSegmenter] Could not parse boundary JSON: ${raw.substring(0, 200)}`);
+            return null;
+        }
+    }
+    /**
+     * Clamp, dedupe, and sort model output. A hallucinated or out-of-range block
+     * index must never become a chunk offset, so anything unusable is dropped here.
+     */
+    sanitizeBoundaries(response, blockCount) {
+        const raw = response?.boundaries ?? [];
+        const seen = new Set();
+        const cleaned = [];
+        for (const entry of raw) {
+            const index = Number(entry?.startBlock);
+            if (!Number.isInteger(index) || index < 0 || index >= blockCount || seen.has(index)) {
+                continue;
+            }
+            seen.add(index);
+            cleaned.push({ startBlock: index, title: entry.title?.trim() || undefined });
+        }
+        cleaned.sort((a, b) => a.startBlock - b.startBlock);
+        if (cleaned.length > 0 && cleaned[0].startBlock !== 0) {
+            cleaned.unshift({ startBlock: 0 });
+        }
+        return cleaned;
+    }
+    // ─────────────────────────────────────────────
+    // Segment assembly
+    // ─────────────────────────────────────────────
+    /** Join blocks between consecutive boundaries into one segment each. */
+    buildSegments(blocks, boundaries) {
+        return boundaries.map((boundary, i) => {
+            const endBlock = i + 1 < boundaries.length ? boundaries[i + 1].startBlock : blocks.length;
+            const slice = blocks.slice(boundary.startBlock, endBlock);
+            const body = slice.map((b) => b.Text).join('\n\n');
+            return {
+                Modality: 'text',
+                Title: boundary.title,
+                Text: boundary.title ? `${boundary.title}\n${body}` : body,
+                StartOffset: slice[0]?.StartOffset,
+                EndOffset: slice[slice.length - 1]?.EndOffset,
+            };
+        });
+    }
+    /** Degrade to structural segmentation — never fail an ingestion run over segmentation. */
+    async fallback(params) {
+        const structural = new StructuralTextSegmenter();
+        const result = await structural.Segment({ Text: params.Text, MimeType: params.MimeType, Options: params.Options });
+        return result.Segments.map((s) => ({
+            Modality: s.Modality,
+            Text: s.Text,
+            Title: s.Title,
+            StartOffset: s.StartOffset,
+            EndOffset: s.EndOffset,
+        }));
+    }
+};
+SemanticTextSegmenter = __decorate([
+    RegisterClass(BaseSegmenter, SEMANTIC_TEXT_SEGMENTER_KEY)
+], SemanticTextSegmenter);
+export { SemanticTextSegmenter };
+//# sourceMappingURL=SemanticTextSegmenter.js.map