@gixcopilot/knowledge 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ahmed khaled
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,44 @@
+# @gixcopilot/knowledge
+
+Framework-independent knowledge source, document, and loader contracts for the AI Copilot SDK —
+files, text, Markdown, PDF, DOCX, HTML/web, APIs, databases, object storage, and MCP resources,
+each normalized into the same `KnowledgeDocument` shape. No chunking, embedding, or vector-store
+concern lives here — see `@gixcopilot/rag`.
+
+## Install
+
+```bash
+npm install @gixcopilot/knowledge
+```
+
+Requires Node.js >=22.12.0. ESM only.
+
+```ts
+import { pdfSource, defaultKnowledgeLoaders } from '@gixcopilot/knowledge';
+
+const source = pdfSource({ id: 'handbook', path: './handbook.pdf', tenantId: 'tenant-a', acl: { permissions: ['knowledge.hr.read'] } });
+for await (const doc of defaultKnowledgeLoaders.pdf!.load(source)) {
+  console.log(doc.metadata.page, doc.content.slice(0, 80));
+}
+```
+
+Attach `tenantId`/`acl` at the source (or per-document) level — that metadata is what
+`@gixcopilot/rag`'s retriever later enforces against a trusted `SecurityContext`. The `web` loader
+refuses loopback/link-local/private-range/non-HTTP(S) URLs and redirects; it is not a general
+model-controlled fetch tool. `database`/`object-storage`/`mcp-resource` sources take a
+host-supplied client or predefined safe query — never natural-language-to-SQL or an arbitrary
+cloud SDK dependency.
+
+See [Phase 9 API](../../docs/phases/phase-09/Phase_9_API.md),
+[architecture](../../docs/phases/phase-09/Phase_9_Architecture.md), and
+[limits](../../docs/phases/phase-09/Phase_9_Issues.md).
+
+## Documentation
+
+- [rag guide](https://github.com/AfoudaSWE/gix_ai_copilot_nx/blob/main/docs/guides/rag.md)
+- [Getting started](https://github.com/AfoudaSWE/gix_ai_copilot_nx/blob/main/docs/guides/getting-started.md)
+- [Source](https://github.com/AfoudaSWE/gix_ai_copilot_nx/tree/main/packages/knowledge)
+
+## License
+
+MIT

package/dist/acl.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Access-control metadata attached to a knowledge source/document at ingestion time (Section
+ * 60). This is *descriptive* metadata only - it participates in deterministic authorization
+ * inside @gixcopilot/rag's retriever (query-time, against a trusted SecurityContext), never
+ * enforced here. A knowledge/document with no ACL is effectively unrestricted within its
+ * tenant - callers that want to restrict a source must say so explicitly.
+ */
+export interface KnowledgeACL {
+    readonly users?: readonly string[];
+    readonly roles?: readonly string[];
+    readonly permissions?: readonly string[];
+    readonly groups?: readonly string[];
+}
+/** An ACL with no populated field grants no additional restriction beyond tenant scoping. */
+export declare function isUnrestrictedAcl(acl: KnowledgeACL | undefined): boolean;
+//# sourceMappingURL=acl.d.ts.map

package/dist/acl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"acl.d.ts","sourceRoot":"","sources":["../src/acl.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACrC;AAMD,6FAA6F;AAC7F,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,YAAY,GAAG,SAAS,GAAG,OAAO,CAGxE"}

package/dist/acl.js

@@ -0,0 +1,10 @@
+function isNonEmpty(values) {
+    return values !== undefined && values.length > 0;
+}
+/** An ACL with no populated field grants no additional restriction beyond tenant scoping. */
+export function isUnrestrictedAcl(acl) {
+    if (!acl)
+        return true;
+    return !isNonEmpty(acl.users) && !isNonEmpty(acl.roles) && !isNonEmpty(acl.permissions) && !isNonEmpty(acl.groups);
+}
+//# sourceMappingURL=acl.js.map

package/dist/acl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"acl.js","sourceRoot":"","sources":["../src/acl.ts"],"names":[],"mappings":"AAcA,SAAS,UAAU,CAAC,MAAqC;IACvD,OAAO,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;AACnD,CAAC;AAED,6FAA6F;AAC7F,MAAM,UAAU,iBAAiB,CAAC,GAA6B;IAC7D,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AACrH,CAAC"}

package/dist/document.d.ts

@@ -0,0 +1,59 @@
+import type { KnowledgeACL } from './acl.js';
+/**
+ * Descriptive metadata about a KnowledgeDocument (Section 12). Free-form `tags`/business
+ * metadata are allowed, but nothing here is implicitly trusted for authorization - only
+ * `tenantId`/`acl` participate in the rag package's retrieval-time authorization checks.
+ * Arbitrary sensitive metadata reaching the model automatically is avoided by never including
+ * this object wholesale in model context - rag's citation/context formatting picks specific
+ * fields deliberately (Section 12).
+ */
+export interface DocumentMetadata {
+    readonly title?: string;
+    /** Where this document originally came from - a file path, URL, or opaque source-specific id. */
+    readonly uri?: string;
+    readonly mimeType?: string;
+    readonly createdAt?: string;
+    readonly updatedAt?: string;
+    readonly author?: string;
+    /** Mandatory for any document that should be tenant-isolated during retrieval (Section 45/59). */
+    readonly tenantId?: string;
+    readonly acl?: KnowledgeACL;
+    readonly tags?: readonly string[];
+    /** 1-based page number, when the source is paginated (PDF, etc.) - Section 14/18. */
+    readonly page?: number;
+    readonly section?: string;
+    readonly heading?: string;
+    /**
+     * Stable digest of `content`, used for change detection/deduplication/idempotent reindexing
+     * (Section 132/133) - always populated by `hashContent`, never left to the caller to compute
+     * inconsistently.
+     */
+    readonly contentHash?: string;
+    readonly businessMetadata?: Readonly<Record<string, unknown>>;
+}
+/** Normalized document representation every loader produces (Section 12). */
+export interface KnowledgeDocument {
+    readonly id: string;
+    readonly sourceId: string;
+    readonly content: string;
+    readonly metadata: DocumentMetadata;
+}
+/** SHA-256 digest of `content`, used for both `contentHash` and deterministic id derivation. */
+export declare function hashContent(content: string): string;
+/**
+ * Deterministic document id: stable across re-ingestion of the *same* source position (so
+ * reindex/update/delete/citation all agree - Section 13), but changes when content changes so
+ * a content-hash-based `contentHash` can still detect the change (Section 132). `discriminator`
+ * disambiguates multiple documents from one source (e.g. a page number, a row id, a resource
+ * URI) - omit only when the source itself is single-document.
+ */
+export declare function deriveDocumentId(sourceId: string, discriminator?: string | number): string;
+export interface CreateKnowledgeDocumentInput {
+    readonly sourceId: string;
+    readonly content: string;
+    readonly discriminator?: string | number;
+    readonly metadata?: Omit<DocumentMetadata, 'contentHash'>;
+}
+/** Builds a KnowledgeDocument with a deterministic id and a populated contentHash. */
+export declare function createKnowledgeDocument(input: CreateKnowledgeDocumentInput): KnowledgeDocument;
+//# sourceMappingURL=document.d.ts.map

package/dist/document.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"document.d.ts","sourceRoot":"","sources":["../src/document.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C;;;;;;;GAOG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,iGAAiG;IACjG,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,kGAAkG;IAClG,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,CAAC,EAAE,YAAY,CAAC;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,qFAAqF;IACrF,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CAC/D;AAED,6EAA6E;AAC7E,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;CACrC;AAED,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAG1F;AAED,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACzC,QAAQ,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;CAC3D;AAED,sFAAsF;AACtF,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,4BAA4B,GAAG,iBAAiB,CAU9F"}

package/dist/document.js

@@ -0,0 +1,29 @@
+import { createHash } from 'node:crypto';
+/** SHA-256 digest of `content`, used for both `contentHash` and deterministic id derivation. */
+export function hashContent(content) {
+    return createHash('sha256').update(content, 'utf8').digest('hex');
+}
+/**
+ * Deterministic document id: stable across re-ingestion of the *same* source position (so
+ * reindex/update/delete/citation all agree - Section 13), but changes when content changes so
+ * a content-hash-based `contentHash` can still detect the change (Section 132). `discriminator`
+ * disambiguates multiple documents from one source (e.g. a page number, a row id, a resource
+ * URI) - omit only when the source itself is single-document.
+ */
+export function deriveDocumentId(sourceId, discriminator) {
+    const key = JSON.stringify([sourceId, discriminator ?? null]);
+    return createHash('sha256').update(key, 'utf8').digest('hex').slice(0, 32);
+}
+/** Builds a KnowledgeDocument with a deterministic id and a populated contentHash. */
+export function createKnowledgeDocument(input) {
+    return {
+        id: deriveDocumentId(input.sourceId, input.discriminator),
+        sourceId: input.sourceId,
+        content: input.content,
+        metadata: {
+            ...input.metadata,
+            contentHash: hashContent(input.content),
+        },
+    };
+}
+//# sourceMappingURL=document.js.map

package/dist/document.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document.js","sourceRoot":"","sources":["../src/document.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AA4CzC,gGAAgG;AAChG,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACpE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB,EAAE,aAA+B;IAChF,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC;IAC9D,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC7E,CAAC;AASD,sFAAsF;AACtF,MAAM,UAAU,uBAAuB,CAAC,KAAmC;IACzE,OAAO;QACL,EAAE,EAAE,gBAAgB,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,aAAa,CAAC;QACzD,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,QAAQ,EAAE;YACR,GAAG,KAAK,CAAC,QAAQ;YACjB,WAAW,EAAE,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC;SACxC;KACF,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,32 @@
+export type { KnowledgeACL } from './acl.js';
+export { isUnrestrictedAcl } from './acl.js';
+export type { DocumentMetadata, KnowledgeDocument, CreateKnowledgeDocumentInput } from './document.js';
+export { hashContent, deriveDocumentId, createKnowledgeDocument } from './document.js';
+export type { KnowledgeSourceType, KnowledgeSource, KnowledgeSourceCommonInput, TextSourceConfig, TextSourceInput, MarkdownSourceInput, BinarySourceConfig, PdfSourceInput, DocxSourceInput, HtmlSourceConfig, HtmlSourceInput, WebSourceConfig, WebSourceInput, DerivedDocumentInput, ApiSourceConfig, ApiSourceInput, DatabaseSourceConfig, DatabaseSourceInput, ObjectStorageObject, ObjectStorageClient, ObjectStorageSourceConfig, ObjectStorageSourceInput, McpResourceReader, McpResourceSourceConfig, McpResourceSourceInput, FileSourceConfig, FileSourceInput, CustomSourceInput, } from './source.js';
+export { textSource, markdownSource, pdfSource, docxSource, htmlSource, webSource, apiSource, createDatabaseKnowledgeSource, objectStorageSource, mcpResourceSource, fileSource, customSource, } from './source.js';
+export type { KnowledgeSourceRegistry } from './registry.js';
+export { createKnowledgeSourceRegistry } from './registry.js';
+export { normalizeText } from './normalize.js';
+export type { DocumentLoader, LoaderContext } from './loader.js';
+export { throwIfAborted } from './loader.js';
+export { textLoader } from './loaders/text.js';
+export { markdownLoader, extractMarkdownTitle } from './loaders/markdown.js';
+export { pdfLoader } from './loaders/pdf.js';
+export { docxLoader } from './loaders/docx.js';
+export { htmlLoader } from './loaders/html.js';
+export { fileLoader } from './loaders/file.js';
+export { webLoader } from './loaders/web.js';
+export { apiLoader } from './loaders/api.js';
+export { databaseLoader, createDatabaseLoader } from './loaders/database.js';
+export { objectStorageLoader } from './loaders/object-storage.js';
+export { mcpResourceLoader } from './loaders/mcp-resource.js';
+export { extractHtmlStructure, serializeTable } from './loaders/html-structure.js';
+export { assertSafeWebUrl, SsrfGuardError } from './net/ssrf-guard.js';
+/**
+ * Default loader lookup by KnowledgeSourceType - `custom` has no default (Section 10: the
+ * caller supplies their own `DocumentLoader` for it).
+ */
+import type { DocumentLoader } from './loader.js';
+import type { KnowledgeSourceType } from './source.js';
+export declare const defaultKnowledgeLoaders: Readonly<Partial<Record<KnowledgeSourceType, DocumentLoader<never>>>>;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAE7C,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,4BAA4B,EAAE,MAAM,eAAe,CAAC;AACvG,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAEvF,YAAY,EACV,mBAAmB,EACnB,eAAe,EACf,0BAA0B,EAC1B,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EACnB,kBAAkB,EAClB,cAAc,EACd,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,yBAAyB,EACzB,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,sBAAsB,EACtB,gBAAgB,EAChB,eAAe,EACf,iBAAiB,GAClB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,UAAU,EACV,cAAc,EACd,SAAS,EACT,UAAU,EACV,UAAU,EACV,SAAS,EACT,SAAS,EACT,6BAA6B,EAC7B,mBAAmB,EACnB,iBAAiB,EACjB,UAAU,EACV,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,6BAA6B,EAAE,MAAM,eAAe,CAAC;AAE9D,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAEnF,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAEvE;;;GAGG;AACH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAavD,eAAO,MAAM,uBAAuB,EAAE,QAAQ,CAC5C,OAAO,CAAC,MAAM,CAAC,mBAAmB,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAa5D,CAAC"}

package/dist/index.js

@@ -0,0 +1,44 @@
+export { isUnrestrictedAcl } from './acl.js';
+export { hashContent, deriveDocumentId, createKnowledgeDocument } from './document.js';
+export { textSource, markdownSource, pdfSource, docxSource, htmlSource, webSource, apiSource, createDatabaseKnowledgeSource, objectStorageSource, mcpResourceSource, fileSource, customSource, } from './source.js';
+export { createKnowledgeSourceRegistry } from './registry.js';
+export { normalizeText } from './normalize.js';
+export { throwIfAborted } from './loader.js';
+export { textLoader } from './loaders/text.js';
+export { markdownLoader, extractMarkdownTitle } from './loaders/markdown.js';
+export { pdfLoader } from './loaders/pdf.js';
+export { docxLoader } from './loaders/docx.js';
+export { htmlLoader } from './loaders/html.js';
+export { fileLoader } from './loaders/file.js';
+export { webLoader } from './loaders/web.js';
+export { apiLoader } from './loaders/api.js';
+export { databaseLoader, createDatabaseLoader } from './loaders/database.js';
+export { objectStorageLoader } from './loaders/object-storage.js';
+export { mcpResourceLoader } from './loaders/mcp-resource.js';
+export { extractHtmlStructure, serializeTable } from './loaders/html-structure.js';
+export { assertSafeWebUrl, SsrfGuardError } from './net/ssrf-guard.js';
+import { textLoader as _text } from './loaders/text.js';
+import { markdownLoader as _markdown } from './loaders/markdown.js';
+import { pdfLoader as _pdf } from './loaders/pdf.js';
+import { docxLoader as _docx } from './loaders/docx.js';
+import { htmlLoader as _html } from './loaders/html.js';
+import { fileLoader as _file } from './loaders/file.js';
+import { webLoader as _web } from './loaders/web.js';
+import { apiLoader as _api } from './loaders/api.js';
+import { databaseLoader as _database } from './loaders/database.js';
+import { objectStorageLoader as _objectStorage } from './loaders/object-storage.js';
+import { mcpResourceLoader as _mcpResource } from './loaders/mcp-resource.js';
+export const defaultKnowledgeLoaders = {
+    text: _text,
+    markdown: _markdown,
+    pdf: _pdf,
+    docx: _docx,
+    html: _html,
+    file: _file,
+    web: _web,
+    api: _api,
+    database: _database,
+    'object-storage': _objectStorage,
+    'mcp-resource': _mcpResource,
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAG7C,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAgCvF,OAAO,EACL,UAAU,EACV,cAAc,EACd,SAAS,EACT,UAAU,EACV,UAAU,EACV,SAAS,EACT,SAAS,EACT,6BAA6B,EAC7B,mBAAmB,EACnB,iBAAiB,EACjB,UAAU,EACV,YAAY,GACb,MAAM,aAAa,CAAC;AAGrB,OAAO,EAAE,6BAA6B,EAAE,MAAM,eAAe,CAAC;AAE9D,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAEnF,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAQvE,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,cAAc,IAAI,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,SAAS,IAAI,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,SAAS,IAAI,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,SAAS,IAAI,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,cAAc,IAAI,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,mBAAmB,IAAI,cAAc,EAAE,MAAM,6BAA6B,CAAC;AACpF,OAAO,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAE9E,MAAM,CAAC,MAAM,uBAAuB,GAEhC;IACF,IAAI,EAAE,KAAK;IACX,QAAQ,EAAE,SAAS;IACnB,GAAG,EAAE,IAAI;IACT,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK;IACX,GAAG,EAAE,IAAI;IACT,GAAG,EAAE,IAAI;IACT,QAAQ,EAAE,SAAS;IACnB,gBAAgB,EAAE,cAAc;IAChC,cAAc,EAAE,YAAY;CAC7B,CAAC"}

package/dist/loader.d.ts

@@ -0,0 +1,14 @@
+import type { KnowledgeDocument } from './document.js';
+import type { KnowledgeSource } from './source.js';
+export interface LoaderContext {
+    readonly signal?: AbortSignal;
+}
+/**
+ * Loads documents from a KnowledgeSource (Section 15). Async iteration so a large source never
+ * has to be fully materialized in memory before the first document is available.
+ */
+export interface DocumentLoader<TConfig = unknown> {
+    load(source: KnowledgeSource<TConfig>, context?: LoaderContext): AsyncIterable<KnowledgeDocument>;
+}
+export declare function throwIfAborted(signal: AbortSignal | undefined): void;
+//# sourceMappingURL=loader.d.ts.map

package/dist/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../src/loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc,CAAC,OAAO,GAAG,OAAO;IAC/C,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAAC;CACnG;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,IAAI,CAIpE"}

package/dist/loader.js

@@ -0,0 +1,6 @@
+export function throwIfAborted(signal) {
+    if (signal?.aborted) {
+        throw new DOMException('The load was aborted.', 'AbortError');
+    }
+}
+//# sourceMappingURL=loader.js.map

package/dist/loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.js","sourceRoot":"","sources":["../src/loader.ts"],"names":[],"mappings":"AAeA,MAAM,UAAU,cAAc,CAAC,MAA+B;IAC5D,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,YAAY,CAAC,uBAAuB,EAAE,YAAY,CAAC,CAAC;IAChE,CAAC;AACH,CAAC"}

package/dist/loaders/api.d.ts

@@ -0,0 +1,10 @@
+import { type DocumentLoader } from '../loader.js';
+import type { ApiSourceConfig } from '../source.js';
+/**
+ * API knowledge loader (Section 21) - intentionally distinct from a Phase 8 API *tool*: it only
+ * turns an already-fetched response into documents, never performs an action. `request`/
+ * `toDocuments` are entirely caller-supplied, so this package owns no HTTP client, base-URL
+ * policy, or credentials.
+ */
+export declare const apiLoader: DocumentLoader<ApiSourceConfig>;
+//# sourceMappingURL=api.d.ts.map

package/dist/loaders/api.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../../src/loaders/api.ts"],"names":[],"mappings":"AAGA,OAAO,EAAkB,KAAK,cAAc,EAAsB,MAAM,cAAc,CAAC;AAEvF,OAAO,KAAK,EAAE,eAAe,EAAmB,MAAM,cAAc,CAAC;AAErE;;;;;GAKG;AACH,eAAO,MAAM,SAAS,EAAE,cAAc,CAAC,eAAe,CAgCrD,CAAC"}

package/dist/loaders/api.js

@@ -0,0 +1,42 @@
+import { CopilotError } from '@gixcopilot/protocol';
+import { createKnowledgeDocument } from '../document.js';
+import { throwIfAborted } from '../loader.js';
+import { normalizeText } from '../normalize.js';
+/**
+ * API knowledge loader (Section 21) - intentionally distinct from a Phase 8 API *tool*: it only
+ * turns an already-fetched response into documents, never performs an action. `request`/
+ * `toDocuments` are entirely caller-supplied, so this package owns no HTTP client, base-URL
+ * policy, or credentials.
+ */
+export const apiLoader = {
+    async *load(source, context) {
+        throwIfAborted(context?.signal);
+        let response;
+        try {
+            response = await source.config.request();
+        }
+        catch (error) {
+            throw CopilotError.sourceLoadFailed(`API request failed: ${String(error)}`, {
+                sourceId: source.id,
+            });
+        }
+        throwIfAborted(context?.signal);
+        const documents = source.config.toDocuments(response);
+        for (const doc of documents) {
+            const content = normalizeText(doc.content);
+            if (content.length === 0)
+                continue;
+            yield createKnowledgeDocument({
+                sourceId: source.id,
+                discriminator: doc.discriminator,
+                content,
+                metadata: {
+                    ...doc.metadata,
+                    tenantId: source.tenantId,
+                    acl: doc.metadata?.acl ?? source.acl,
+                },
+            });
+        }
+    },
+};
+//# sourceMappingURL=api.js.map

package/dist/loaders/api.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"api.js","sourceRoot":"","sources":["../../src/loaders/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAA2C,MAAM,cAAc,CAAC;AACvF,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,SAAS,GAAoC;IACxD,KAAK,CAAC,CAAC,IAAI,CACT,MAAwC,EACxC,OAAuB;QAEvB,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAChC,IAAI,QAAiB,CAAC;QACtB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC3C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,CAAC,gBAAgB,CAAC,uBAAuB,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE;gBAC1E,QAAQ,EAAE,MAAM,CAAC,EAAE;aACpB,CAAC,CAAC;QACL,CAAC;QAED,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAChC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACtD,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YACnC,MAAM,uBAAuB,CAAC;gBAC5B,QAAQ,EAAE,MAAM,CAAC,EAAE;gBACnB,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,OAAO;gBACP,QAAQ,EAAE;oBACR,GAAG,GAAG,CAAC,QAAQ;oBACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,GAAG,IAAI,MAAM,CAAC,GAAG;iBACrC;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF,CAAC"}

package/dist/loaders/database.d.ts

@@ -0,0 +1,12 @@
+import { type DocumentLoader } from '../loader.js';
+import type { DatabaseSourceConfig } from '../source.js';
+/**
+ * Database knowledge loader (Section 22). `query` is the caller's own already-parameterized
+ * data access - this loader never constructs or executes SQL, so unrestricted
+ * natural-language-to-SQL access is impossible by construction (Section 22's explicit
+ * prohibition).
+ */
+export declare function createDatabaseLoader<TRow = unknown>(): DocumentLoader<DatabaseSourceConfig<TRow>>;
+/** Default-configured database loader for callers not using a custom row type. */
+export declare const databaseLoader: DocumentLoader<DatabaseSourceConfig>;
+//# sourceMappingURL=database.d.ts.map

package/dist/loaders/database.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../src/loaders/database.ts"],"names":[],"mappings":"AAGA,OAAO,EAAkB,KAAK,cAAc,EAAsB,MAAM,cAAc,CAAC;AAEvF,OAAO,KAAK,EAAE,oBAAoB,EAAmB,MAAM,cAAc,CAAC;AAE1E;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,GAAG,OAAO,KAAK,cAAc,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAkCjG;AAED,kFAAkF;AAClF,eAAO,MAAM,cAAc,EAAE,cAAc,CAAC,oBAAoB,CAA0B,CAAC"}

package/dist/loaders/database.js

@@ -0,0 +1,46 @@
+import { CopilotError } from '@gixcopilot/protocol';
+import { createKnowledgeDocument } from '../document.js';
+import { throwIfAborted } from '../loader.js';
+import { normalizeText } from '../normalize.js';
+/**
+ * Database knowledge loader (Section 22). `query` is the caller's own already-parameterized
+ * data access - this loader never constructs or executes SQL, so unrestricted
+ * natural-language-to-SQL access is impossible by construction (Section 22's explicit
+ * prohibition).
+ */
+export function createDatabaseLoader() {
+    return {
+        async *load(source, context) {
+            throwIfAborted(context?.signal);
+            let rows;
+            try {
+                rows = await source.config.query();
+            }
+            catch (error) {
+                throw CopilotError.sourceLoadFailed(`Database query failed: ${String(error)}`, {
+                    sourceId: source.id,
+                });
+            }
+            throwIfAborted(context?.signal);
+            for (const [index, row] of rows.entries()) {
+                const doc = source.config.toDocument(row, index);
+                const content = normalizeText(doc.content);
+                if (content.length === 0)
+                    continue;
+                yield createKnowledgeDocument({
+                    sourceId: source.id,
+                    discriminator: doc.discriminator ?? index,
+                    content,
+                    metadata: {
+                        ...doc.metadata,
+                        tenantId: source.tenantId,
+                        acl: doc.metadata?.acl ?? source.acl,
+                    },
+                });
+            }
+        },
+    };
+}
+/** Default-configured database loader for callers not using a custom row type. */
+export const databaseLoader = createDatabaseLoader();
+//# sourceMappingURL=database.js.map

package/dist/loaders/database.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"database.js","sourceRoot":"","sources":["../../src/loaders/database.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAA2C,MAAM,cAAc,CAAC;AACvF,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhD;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB;IAClC,OAAO;QACL,KAAK,CAAC,CAAC,IAAI,CACT,MAAmD,EACnD,OAAuB;YAEvB,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAChC,IAAI,IAAqB,CAAC;YAC1B,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACrC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,YAAY,CAAC,gBAAgB,CAAC,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE;oBAC7E,QAAQ,EAAE,MAAM,CAAC,EAAE;iBACpB,CAAC,CAAC;YACL,CAAC;YAED,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAChC,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBACjD,MAAM,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBACnC,MAAM,uBAAuB,CAAC;oBAC5B,QAAQ,EAAE,MAAM,CAAC,EAAE;oBACnB,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,KAAK;oBACzC,OAAO;oBACP,QAAQ,EAAE;wBACR,GAAG,GAAG,CAAC,QAAQ;wBACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,GAAG,IAAI,MAAM,CAAC,GAAG;qBACrC;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,kFAAkF;AAClF,MAAM,CAAC,MAAM,cAAc,GAAyC,oBAAoB,EAAE,CAAC"}

package/dist/loaders/docx.d.ts

@@ -0,0 +1,9 @@
+import { type DocumentLoader } from '../loader.js';
+import type { BinarySourceConfig } from '../source.js';
+/**
+ * DOCX loader (Section 19) - converts to HTML via mammoth, then reuses the HTML structure
+ * extractor to preserve headings/paragraphs/tables (Section 20's table serialization applies
+ * here too, since mammoth already renders DOCX tables as `<table>`).
+ */
+export declare const docxLoader: DocumentLoader<BinarySourceConfig>;
+//# sourceMappingURL=docx.d.ts.map

package/dist/loaders/docx.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"docx.d.ts","sourceRoot":"","sources":["../../src/loaders/docx.ts"],"names":[],"mappings":"AAKA,OAAO,EAAkB,KAAK,cAAc,EAAsB,MAAM,cAAc,CAAC;AAEvF,OAAO,KAAK,EAAE,kBAAkB,EAAmB,MAAM,cAAc,CAAC;AASxE;;;;GAIG;AACH,eAAO,MAAM,UAAU,EAAE,cAAc,CAAC,kBAAkB,CAwCzD,CAAC"}

package/dist/loaders/docx.js

@@ -0,0 +1,58 @@
+import { readFile } from 'node:fs/promises';
+import { CopilotError } from '@gixcopilot/protocol';
+import { convertToHtml } from 'mammoth';
+import { createKnowledgeDocument } from '../document.js';
+import { throwIfAborted } from '../loader.js';
+import { normalizeText } from '../normalize.js';
+import { extractHtmlStructure } from './html-structure.js';
+async function readDocxBuffer(config) {
+    if (config.buffer)
+        return config.buffer;
+    if (config.path)
+        return readFile(config.path);
+    throw CopilotError.sourceLoadFailed('A DOCX source requires either "path" or "buffer".');
+}
+/**
+ * DOCX loader (Section 19) - converts to HTML via mammoth, then reuses the HTML structure
+ * extractor to preserve headings/paragraphs/tables (Section 20's table serialization applies
+ * here too, since mammoth already renders DOCX tables as `<table>`).
+ */
+export const docxLoader = {
+    async *load(source, context) {
+        throwIfAborted(context?.signal);
+        const buffer = await readDocxBuffer(source.config).catch((error) => {
+            if (CopilotError.isCopilotError(error))
+                throw error;
+            throw CopilotError.sourceLoadFailed(`Failed to read DOCX source: ${String(error)}`, {
+                sourceId: source.id,
+            });
+        });
+        let html;
+        try {
+            html = (await convertToHtml({ buffer })).value;
+        }
+        catch (error) {
+            throw CopilotError.parseFailed(`Failed to parse DOCX source: ${String(error)}`, {
+                sourceId: source.id,
+            });
+        }
+        throwIfAborted(context?.signal);
+        const structure = extractHtmlStructure(html);
+        const content = normalizeText(structure.text);
+        if (content.length === 0)
+            return;
+        yield createKnowledgeDocument({
+            sourceId: source.id,
+            content,
+            metadata: {
+                ...(source.name !== undefined || structure.title !== undefined
+                    ? { title: source.name ?? structure.title }
+                    : {}),
+                mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+                tenantId: source.tenantId,
+                acl: source.acl,
+            },
+        });
+    },
+};
+//# sourceMappingURL=docx.js.map

package/dist/loaders/docx.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"docx.js","sourceRoot":"","sources":["../../src/loaders/docx.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAA2C,MAAM,cAAc,CAAC;AACvF,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAE3D,KAAK,UAAU,cAAc,CAAC,MAA0B;IACtD,IAAI,MAAM,CAAC,MAAM;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC;IACxC,IAAI,MAAM,CAAC,IAAI;QAAE,OAAO,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC9C,MAAM,YAAY,CAAC,gBAAgB,CAAC,mDAAmD,CAAC,CAAC;AAC3F,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAuC;IAC5D,KAAK,CAAC,CAAC,IAAI,CACT,MAA2C,EAC3C,OAAuB;QAEvB,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;YAC1E,IAAI,YAAY,CAAC,cAAc,CAAC,KAAK,CAAC;gBAAE,MAAM,KAAK,CAAC;YACpD,MAAM,YAAY,CAAC,gBAAgB,CAAC,+BAA+B,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE;gBAClF,QAAQ,EAAE,MAAM,CAAC,EAAE;aACpB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,IAAY,CAAC;QACjB,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,MAAM,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QACjD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,CAAC,WAAW,CAAC,gCAAgC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE;gBAC9E,QAAQ,EAAE,MAAM,CAAC,EAAE;aACpB,CAAC,CAAC;QACL,CAAC;QAED,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAChC,MAAM,SAAS,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEjC,MAAM,uBAAuB,CAAC;YAC5B,QAAQ,EAAE,MAAM,CAAC,EAAE;YACnB,OAAO;YACP,QAAQ,EAAE;gBACR,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,SAAS,CAAC,KAAK,KAAK,SAAS;oBAC5D,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE;oBAC3C,CAAC,CAAC,EAAE,CAAC;gBACP,QAAQ,EAAE,yEAAyE;gBACnF,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,GAAG,EAAE,MAAM,CAAC,GAAG;aAChB;SACF,CAAC,CAAC;IACL,CAAC;CACF,CAAC"}

package/dist/loaders/file.d.ts

@@ -0,0 +1,5 @@
+import { type DocumentLoader } from '../loader.js';
+import type { FileSourceConfig } from '../source.js';
+/** Generic file source (Section 10's `file` type) - dispatches to the matching typed loader by extension. */
+export declare const fileLoader: DocumentLoader<FileSourceConfig>;
+//# sourceMappingURL=file.d.ts.map

package/dist/loaders/file.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file.d.ts","sourceRoot":"","sources":["../../src/loaders/file.ts"],"names":[],"mappings":"AAIA,OAAO,EAAkB,KAAK,cAAc,EAAsB,MAAM,cAAc,CAAC;AAEvF,OAAO,KAAK,EAAE,gBAAgB,EAAmB,MAAM,cAAc,CAAC;AAQtE,6GAA6G;AAC7G,eAAO,MAAM,UAAU,EAAE,cAAc,CAAC,gBAAgB,CAgDvD,CAAC"}

package/dist/loaders/file.js

@@ -0,0 +1,56 @@
+import { extname } from 'node:path';
+import { readFile } from 'node:fs/promises';
+import { createKnowledgeDocument } from '../document.js';
+import { throwIfAborted } from '../loader.js';
+import { normalizeText } from '../normalize.js';
+import { docxLoader } from './docx.js';
+import { htmlLoader } from './html.js';
+import { markdownLoader } from './markdown.js';
+import { pdfLoader } from './pdf.js';
+const EXTENSION_DISPATCH = new Set(['.pdf', '.docx', '.html', '.htm', '.md', '.markdown']);
+/** Generic file source (Section 10's `file` type) - dispatches to the matching typed loader by extension. */
+export const fileLoader = {
+    async *load(source, context) {
+        throwIfAborted(context?.signal);
+        const { path } = source.config;
+        const base = {
+            id: source.id,
+            ...(source.name !== undefined ? { name: source.name } : {}),
+            ...(source.tenantId !== undefined ? { tenantId: source.tenantId } : {}),
+            ...(source.acl !== undefined ? { acl: source.acl } : {}),
+            ...(source.metadata !== undefined ? { metadata: source.metadata } : {}),
+        };
+        const ext = extname(path).toLowerCase();
+        if (!EXTENSION_DISPATCH.has(ext)) {
+            const content = await readFile(path, 'utf8');
+            yield createKnowledgeDocument({
+                sourceId: source.id,
+                content: normalizeText(content),
+                metadata: {
+                    ...(source.name !== undefined ? { title: source.name } : {}),
+                    uri: path,
+                    mimeType: 'text/plain',
+                    tenantId: source.tenantId,
+                    acl: source.acl,
+                },
+            });
+            return;
+        }
+        if (ext === '.pdf') {
+            yield* pdfLoader.load({ ...base, type: 'pdf', config: { path } }, context);
+            return;
+        }
+        if (ext === '.docx') {
+            yield* docxLoader.load({ ...base, type: 'docx', config: { path } }, context);
+            return;
+        }
+        if (ext === '.html' || ext === '.htm') {
+            yield* htmlLoader.load({ ...base, type: 'html', config: { path } }, context);
+            return;
+        }
+        // .md / .markdown
+        const content = await readFile(path, 'utf8');
+        yield* markdownLoader.load({ ...base, type: 'markdown', config: { content } }, context);
+    },
+};
+//# sourceMappingURL=file.js.map

package/dist/loaders/file.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file.js","sourceRoot":"","sources":["../../src/loaders/file.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAA2C,MAAM,cAAc,CAAC;AACvF,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC;AAE3F,6GAA6G;AAC7G,MAAM,CAAC,MAAM,UAAU,GAAqC;IAC1D,KAAK,CAAC,CAAC,IAAI,CACT,MAAyC,EACzC,OAAuB;QAEvB,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAChC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;QAC/B,MAAM,IAAI,GAAG;YACX,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE,CAAC;QACF,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QAExC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC7C,MAAM,uBAAuB,CAAC;gBAC5B,QAAQ,EAAE,MAAM,CAAC,EAAE;gBACnB,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC;gBAC/B,QAAQ,EAAE;oBACR,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC5D,GAAG,EAAE,IAAI;oBACT,QAAQ,EAAE,YAAY;oBACtB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,GAAG,EAAE,MAAM,CAAC,GAAG;iBAChB;aACF,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;YACnB,KAAK,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;YAC3E,OAAO;QACT,CAAC;QACD,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;YACpB,KAAK,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;YAC7E,OAAO;QACT,CAAC;QACD,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;YACtC,KAAK,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;YAC7E,OAAO;QACT,CAAC;QACD,kBAAkB;QAClB,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC7C,KAAK,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IAC1F,CAAC;CACF,CAAC"}