@equinor/fusion-framework-cli-plugin-ai-index 1.0.5 → 2.0.0

package/src/utils/markdown/parser.ts

@@ -10,9 +10,8 @@ import type { MarkdownDocument, MarkdownMetadata } from './types.js';
 import { generateChunkId } from '../generate-chunk-id.js';
 
 const markdownConfig = {
-  chunkSize: 3000,
+  chunkSize: 2000,
   chunkOverlap: 300,
-  keepSeparator: true,
   separators: [
     '\n# ',
     '\n## ',
@@ -21,25 +20,52 @@ const markdownConfig = {
     '\n##### ',
     '\n###### ',
     '\n```',
+    '\n```\n',
     '\n---\n',
     '\n\n',
+    '\n',
   ],
+  keepSeparator: true,
 };
 
 /**
- * Check if a file is a markdown or MDX file
- * @param filePath - File path to check
- * @returns True if file has .md or .mdx extension
+ * Checks whether a file path has a Markdown (`.md`) or MDX (`.mdx`) extension.
+ *
+ * @param filePath - Absolute or relative file path.
+ * @returns `true` when the extension is `.md` or `.mdx`.
  */
 export const isMarkdownFile = (filePath: string): boolean => {
   return filePath.endsWith('.md') || filePath.endsWith('.mdx');
 };
 
 /**
- * Parse markdown or MDX content into document chunks
- * @param content - Markdown or MDX content string
- * @param source - Source file path
- * @returns Array of markdown documents
+ * Validates that a text chunk contains meaningful content.
+ *
+ * Filters out empty strings and chunks consisting solely of code-fence
+ * markers (e.g. ` ``` ` or ` ```ts `).
+ *
+ * @param chunk - Content chunk to validate.
+ * @returns `true` if the chunk has substantive content.
+ */
+const isValidChunk = (chunk: string): boolean => {
+  const trimmed = chunk.trim();
+  if (!trimmed) return false;
+  // Skip chunks that are only code fence markers
+  if (/^`{3,}[\w-]*$/.test(trimmed)) return false;
+  return true;
+};
+
+/**
+ * Parses Markdown or MDX content into chunked vector-store documents.
+ *
+ * Extracts YAML frontmatter via `gray-matter`, splits the body using
+ * {@link RecursiveCharacterTextSplitter}, and returns one
+ * {@link MarkdownDocument} per valid chunk.
+ *
+ * @template T - Additional frontmatter attributes.
+ * @param content - Raw Markdown / MDX string.
+ * @param source - Relative source file path used as the document key.
+ * @returns Array of chunked Markdown documents.
  */
 export const parseMarkdown = async <T extends Record<string, unknown> = Record<string, unknown>>(
   content: string,
@@ -57,7 +83,16 @@ export const parseMarkdown = async <T extends Record<string, unknown> = Record<s
   );
   const textSplitter = new RecursiveCharacterTextSplitter(markdownConfig);
   const chunks = await textSplitter.splitText(markdownContent);
-  return chunks.map(
+
+  // Filter out empty chunks and chunks that are just code fence markers
+  const validChunks = chunks.filter((chunk) => {
+    if (!isValidChunk(chunk)) {
+      return false;
+    }
+    return true;
+  });
+
+  return validChunks.map(
     (chunk, _index): MarkdownDocument<T> => ({
       id: generateChunkId(source, _index),
       pageContent: chunk,
@@ -70,9 +105,15 @@ export const parseMarkdown = async <T extends Record<string, unknown> = Record<s
 };
 
 /**
- * Parse a markdown or MDX file into document chunks
- * @param file - Source file object
- * @returns Array of markdown documents with root path metadata
+ * Reads a Markdown or MDX file from disk and parses it into chunked documents.
+ *
+ * Delegates to {@link parseMarkdown} after reading the file content, then
+ * enriches each resulting document with the `rootPath` from the source file.
+ *
+ * @template T - Additional frontmatter attributes.
+ * @param file - Source file descriptor with path and optional project root.
+ * @returns Array of Markdown documents with root-path metadata.
+ * @throws {AssertionError} If the file does not have a `.md` or `.mdx` extension.
  */
 export const parseMarkdownFile = async <
   T extends Record<string, unknown> = Record<string, unknown>,

package/src/utils/markdown/types.ts

@@ -4,17 +4,28 @@ import type {
 } from '@equinor/fusion-framework-module-ai/lib';
 
 /**
- * Markdown document metadata
+ * Metadata shape for documents generated from Markdown / MDX files.
+ *
+ * Extends the base vector-store metadata with a `'markdown'` type discriminator
+ * and any frontmatter key-value pairs (prefixed with `md_`).
+ *
+ * @template T - Additional frontmatter attributes.
  */
 export type MarkdownMetadata<T extends Record<string, unknown> = Record<string, unknown>> =
   VectorStoreDocumentMetadata<
     T & {
+      /** Discriminator identifying the document as extracted from Markdown. */
       type: 'markdown';
     }
   >;
 
 /**
- * Markdown document
+ * A vector-store document originating from a Markdown or MDX file.
+ *
+ * Contains a text chunk of the markdown content together with
+ * {@link MarkdownMetadata}.
+ *
+ * @template T - Additional frontmatter attributes.
  */
 export type MarkdownDocument<T extends Record<string, unknown> = Record<string, unknown>> =
   VectorStoreDocument<MarkdownMetadata<T>>;

package/src/utils/package-resolver.ts

@@ -2,20 +2,24 @@ import { dirname } from 'node:path';
 import { readPackageUp, type PackageJson } from 'read-package-up';
 
 /**
- * Map of package base directories to package.json
+ * In-memory cache mapping package root directories to their parsed `package.json`.
+ * Avoids repeated file-system lookups when processing many files from the same package.
  */
 const packageMap = new Map<string, PackageJson>();
 
 /**
- * Resolves which package a file path belongs to.
- * First checks the cache map, then uses read-package-up if no match found.
+ * Resolves the nearest `package.json` for a given file path.
  *
- * @param filePath - Absolute or relative file path (e.g., '/path/to/packages/cli/src/index.ts')
- * @returns Package.json if found, undefined otherwise
+ * Uses an in-memory cache keyed by the package’s directory to avoid
+ * redundant file-system lookups when many files share the same package.
+ *
+ * @param filePath - Absolute path to a source file.
+ * @returns The parsed `PackageJson` if found, or `undefined`.
  *
  * @example
  * ```ts
- * const packageJson = await resolvePackage('/path/to/packages/cli/src/index.ts');
+ * const pkg = await resolvePackage('/repo/packages/cli/src/index.ts');
+ * console.log(pkg?.name); // '@equinor/fusion-framework-cli'
  * ```
  */
 export async function resolvePackage(filePath: string): Promise<PackageJson | undefined> {

package/src/utils/ts-doc/constants.ts

@@ -1,7 +1,10 @@
 import { SyntaxKind } from 'ts-morph';
 
 /**
- * Supported TSDoc node kinds for top-level processing
+ * Top-level TypeScript syntax kinds that the TSDoc extractor inspects.
+ *
+ * Only nodes matching one of these kinds are considered for document
+ * extraction; all other descendants are skipped.
  */
 export const nodeKinds = [
   SyntaxKind.FunctionDeclaration,

package/src/utils/ts-doc/extractors.ts

@@ -46,11 +46,16 @@ const createTypescriptDocument = (
 };
 
 /**
- * Extracts a TSDoc document from a class node, including TSDoc, constructor, and public member signatures.
- * @param classNode - The ClassDeclaration node to process.
- * @param sourceFile - The source file containing the node.
- * @param options - Optional parsing configuration.
- * @returns A TypeScript document with TSDoc and class interface, or null if no TSDoc is found.
+ * Extracts a vector-store document from a TypeScript class declaration.
+ *
+ * Collects the class-level TSDoc comment, constructor signature (if documented),
+ * and all public members with TSDoc into a single document whose `pageContent`
+ * mirrors a minimal class interface.
+ *
+ * @param classNode - The `ts-morph` {@link ClassDeclaration} node to inspect.
+ * @param sourceFile - The project source file that contains the class.
+ * @param _options - Optional parsing configuration.
+ * @returns A {@link TypescriptDocument}, or `null` when the class has no TSDoc.
  */
 export const extractDocumentFromClassNode = (
   classNode: ClassDeclaration,
@@ -128,12 +133,19 @@ export const extractDocumentFromClassNode = (
 };
 
 /**
- * Extracts a TSDoc document from a single node.
- * @param node - The TypeScript node to process.
- * @param sourceFile - The source file containing the node.
+ * Extracts a vector-store document from a single TypeScript AST node.
+ *
+ * Handles function declarations, variable statements (arrow / function
+ * expressions), interfaces, type aliases, enums, and classes. Delegates
+ * to {@link extractDocumentFromClassNode} for class declarations.
+ *
+ * @param node - The `ts-morph` AST node to inspect.
+ * @param sourceFile - The project source file that contains the node.
  * @param options - Optional parsing configuration.
- * @param nodeOptions - Optional node-specific configuration (e.g., skipKindCheck for VariableStatement).
- * @returns A TypeScript document with TSDoc metadata, or null if no TSDoc is found.
+ * @param nodeOptions - Optional flags (e.g. `skipKindCheck`) to override default
+ *   kind filtering.
+ * @returns A {@link TypescriptDocument}, or `null` when the node has no TSDoc or
+ *   is not a supported kind.
  */
 export const extractDocumentFromNode = (
   node: Node,
@@ -224,10 +236,12 @@ export const extractDocumentFromNode = (
 };
 
 /**
- * Processes a TypeScript source file to extract TSDoc documents.
- * @param sourceFile - The source file to process.
+ * Walks a TypeScript source file and extracts a {@link TypescriptDocument}
+ * for every top-level declaration that carries a TSDoc comment.
+ *
+ * @param sourceFile - The `ts-morph` source file to traverse.
  * @param options - Optional parsing configuration.
- * @returns An array of TypeScript documents with TSDoc metadata.
+ * @returns Array of extracted documents (one per documented declaration).
  */
 export const processSourceFile = (
   sourceFile: ProjectSourceFile,

package/src/utils/ts-doc/parser.ts

@@ -7,19 +7,24 @@ import type { TypescriptDocument, ParseTsDocOptions } from './types.js';
 import { processSourceFile } from './extractors.js';
 
 /**
- * Checks if a file is a TypeScript or TSX file based on its extension.
- * @param filePath - The path to the file.
- * @returns True if the file ends with .ts or .tsx, false otherwise.
+ * Checks whether a file path has a TypeScript (`.ts`) or TSX (`.tsx`) extension.
+ *
+ * @param filePath - Absolute or relative file path.
+ * @returns `true` if the file extension is `.ts` or `.tsx`.
  */
 export const isTypescriptFile = (filePath: string): boolean => {
   return filePath.endsWith('.ts') || filePath.endsWith('.tsx');
 };
 
 /**
- * Parses TSDoc from a string of TypeScript code.
- * @param content - The TypeScript code content.
+ * Parses TSDoc comments from an in-memory TypeScript code string.
+ *
+ * Creates a temporary `ts-morph` project, analyses the source, and returns
+ * one {@link TypescriptDocument} per documented top-level declaration.
+ *
+ * @param content - TypeScript source code to parse.
  * @param options - Optional parsing configuration.
- * @returns An array of TypeScript documents with TSDoc metadata.
+ * @returns Array of extracted TypeScript documents with TSDoc metadata.
  */
 export const parseTsDocSync = (
   content: string,
@@ -31,11 +36,15 @@ export const parseTsDocSync = (
 };
 
 /**
- * Parses TSDoc from a TypeScript file by path.
- * @param file - The source file object.
+ * Parses TSDoc comments from a TypeScript file on disk.
+ *
+ * Reads the file synchronously, creates a `ts-morph` project, and returns
+ * one {@link TypescriptDocument} per documented top-level declaration.
+ *
+ * @param file - Source file descriptor with path and optional project root.
  * @param options - Optional parsing configuration.
- * @returns An array of TypeScript documents with TSDoc metadata.
- * @throws If the file is not a TypeScript file.
+ * @returns Array of extracted TypeScript documents.
+ * @throws {AssertionError} If the file does not have a `.ts` or `.tsx` extension.
  */
 export const parseTsDocFromFileSync = (
   file: SourceFile,

package/src/utils/ts-doc/types.ts

@@ -4,23 +4,31 @@ import type {
 } from '@equinor/fusion-framework-module-ai/lib';
 
 /**
- * TypeScript document metadata
+ * Metadata shape for documents generated from TypeScript source files.
+ *
+ * Extends the base vector-store metadata with TSDoc-specific fields.
  */
 export type TypescriptMetadata = VectorStoreDocumentMetadata<{
+  /** Discriminator identifying the document as extracted from TSDoc. */
   type: 'tsdoc';
+  /** The `ts-morph` syntax-kind name (e.g. `'FunctionDeclaration'`). */
   ts_kind: string;
+  /** Name of the TypeScript symbol (function, class, interface, etc.). */
   ts_name: string;
 }>;
 
 /**
- * TypeScript document with TSDoc metadata
+ * A vector-store document originating from a TypeScript source file.
+ *
+ * Contains the extracted TSDoc comment (and optionally the type signature)
+ * together with {@link TypescriptMetadata}.
  */
 export type TypescriptDocument = VectorStoreDocument<TypescriptMetadata>;
 
 /**
- * Options for parsing TypeScript documents
+ * Options for controlling TypeScript document parsing behaviour.
  */
 export interface ParseTsDocOptions {
-  /** The project root path for generating relative paths */
+  /** Absolute path to the project root, used to compute relative source paths. */
   projectRoot?: string;
 }

package/src/utils/types.ts

@@ -1,18 +1,22 @@
 /**
- * File change status in git
+ * Git-tracked change status of a source file.
+ *
+ * - `'new'` — file is untracked or newly added.
+ * - `'modified'` — file is tracked and has been changed.
+ * - `'removed'` — file has been deleted.
  */
 export type FileStatus = 'new' | 'modified' | 'removed';
 
 /**
- * Source file information for processing
+ * Represents a source file to be indexed, enriched with path and git status info.
  */
 export type SourceFile = {
-  /** Absolute file path */
+  /** Absolute file system path. */
   path: string;
-  /** Project root directory (git repository root) */
+  /** Absolute path to the git repository root. */
   projectRoot?: string;
-  /** Relative path from project root */
+  /** Path relative to {@link projectRoot}. */
   relativePath?: string;
-  /** Git change status */
+  /** Current git change status. */
   status: FileStatus;
 };

package/src/version.ts

@@ -1,2 +1,2 @@
 // Generated by genversion.
-export const version = '1.0.5';
+export const version = '2.0.0';

package/dist/esm/command.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"command.js","sourceRoot":"","sources":["../../src/command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAExD,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AAClG,OAAO,EAAE,WAAW,IAAI,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAE5G,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,oBAAoB,EAAuB,MAAM,sBAAsB,CAAC;AAGjF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAM,QAAQ,GAAG,aAAa,CAAC,YAAY,CAAC;KACzC,WAAW,CAAC,kEAAkE,CAAC;KAC/E,SAAS,CACR,YAAY,CAAC,WAAW,EAAE,wDAAwD,CAAC,CAAC,OAAO,CACzF,KAAK,CACN,CACF;KACA,SAAS,CAAC,YAAY,CAAC,mBAAmB,EAAE,uBAAuB,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;KACjG,SAAS,CAAC,YAAY,CAAC,QAAQ,EAAE,4CAA4C,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;KAC9F,SAAS,CAAC,YAAY,CAAC,kBAAkB,EAAE,kCAAkC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;KACjG,SAAS,CACR,YAAY,CACV,SAAS,EACT,uEAAuE,CACxE,CAAC,OAAO,CAAC,KAAK,CAAC,CACjB;KACA,QAAQ,CAAC,oBAAoB,EAAE,2DAA2D,CAAC;KAC3F,MAAM,CAAC,KAAK,EAAE,QAAkB,EAAE,cAA8B,EAAE,EAAE;IACnE,MAAM,OAAO,GAAG,MAAM,oBAAoB,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;IAEtE,qBAAqB;IACrB,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAA0B,OAAO,CAAC,MAAM,EAAE;QAC/E,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE;KACvB,CAAC,CAAC;IAEH,gDAAgD;IAChD,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IACvC,MAAM,mBAAmB,GAAG,WAAW,CAAC,QAAQ,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IACvF,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,mBAAmB,CAAC;IAEtE,uBAAuB;IACvB,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAEhD,oDAAoD;IACpD,MAAM,KAAK,CAAC;QACV,SAAS;QACT,OAAO;QACP,MAAM;QACN,YAAY;KACb,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,EAAE;IAC7C,gBAAgB,EAAE,IAAI;IACtB,aAAa,EAAE,IAAI;CACpB,CAAC,CAAC;AAEH,eAAe,OAAO,CAAC"}

package/dist/esm/command.options.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"command.options.js","sourceRoot":"","sources":["../../src/command.options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,eAAe,EAAE,MAAM,8DAA8D,CAAC;AAE/F;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,eAAe,CAAC,MAAM,CAAC;IACzD,4EAA4E;IAC5E,yBAAyB,EAAE,CAAC;SACzB,MAAM,CAAC,EAAE,OAAO,EAAE,+DAA+D,EAAE,CAAC;SACpF,GAAG,CAAC,CAAC,EAAE,uDAAuD,CAAC;SAC/D,QAAQ,CAAC,wCAAwC,CAAC;IACrD,mBAAmB,EAAE,CAAC;SACnB,MAAM,CAAC,EAAE,OAAO,EAAE,2DAA2D,EAAE,CAAC;SAChF,GAAG,CAAC,4CAA4C,CAAC;SACjD,GAAG,CAAC,CAAC,EAAE,mDAAmD,CAAC;SAC3D,QAAQ,CAAC,2BAA2B,CAAC;IACxC,iBAAiB,EAAE,CAAC;SACjB,MAAM,CAAC,EAAE,OAAO,EAAE,0DAA0D,EAAE,CAAC;SAC/E,GAAG,CAAC,CAAC,EAAE,kDAAkD,CAAC;SAC1D,QAAQ,CAAC,sBAAsB,CAAC;IACnC,oBAAoB,EAAE,CAAC;SACpB,MAAM,CAAC,EAAE,OAAO,EAAE,6DAA6D,EAAE,CAAC;SAClF,GAAG,CAAC,CAAC,EAAE,qDAAqD,CAAC;SAC7D,QAAQ,CAAC,yBAAyB,CAAC;IAEtC,8BAA8B;IAC9B,MAAM,EAAE,CAAC;SACN,OAAO,CAAC,EAAE,OAAO,EAAE,iCAAiC,EAAE,CAAC;SACvD,QAAQ,CAAC,wDAAwD,CAAC;IACrE,MAAM,EAAE,CAAC;SACN,MAAM,CAAC,EAAE,OAAO,EAAE,8DAA8D,EAAE,CAAC;SACnF,GAAG,CAAC,CAAC,EAAE,8CAA8C,CAAC;SACtD,QAAQ,CAAC,uBAAuB,CAAC;IACpC,IAAI,EAAE,CAAC;SACJ,OAAO,CAAC,EAAE,OAAO,EAAE,+BAA+B,EAAE,CAAC;SACrD,QAAQ,CAAC,4CAA4C,CAAC;IACzD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;IAClF,KAAK,EAAE,CAAC;SACL,OAAO,CAAC,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC;SACtD,QAAQ,CAAC,uEAAuE,CAAC;CACrF,CAAC,CAAC,QAAQ,CAAC,4CAA4C,CAAC,CAAC"}

package/dist/types/command.d.ts

@@ -1,2 +0,0 @@
-export declare const command: import("commander").Command;
-export default command;

package/dist/types/command.options.d.ts

@@ -1,33 +0,0 @@
-import { z } from 'zod';
-/**
- * Zod schema for validating command options for the embeddings command.
- *
- * This schema extends the base AI options schema with embeddings-specific options,
- * ensuring type safety and runtime validation of command arguments.
- *
- * Note: Some optional AI options become required for the embeddings command
- * (openaiEmbeddingDeployment, azureSearchEndpoint, azureSearchApiKey, azureSearchIndexName)
- * because the command uses withAiOptions with includeEmbedding and includeSearch set to true.
- */
-export declare const CommandOptionsSchema: z.ZodObject<{
-    openaiApiKey: z.ZodString;
-    openaiApiVersion: z.ZodString;
-    openaiInstance: z.ZodString;
-    openaiChatDeployment: z.ZodOptional<z.ZodString>;
-    openaiEmbeddingDeployment: z.ZodString;
-    azureSearchEndpoint: z.ZodString;
-    azureSearchApiKey: z.ZodString;
-    azureSearchIndexName: z.ZodString;
-    dryRun: z.ZodBoolean;
-    config: z.ZodString;
-    diff: z.ZodBoolean;
-    baseRef: z.ZodOptional<z.ZodString>;
-    clean: z.ZodBoolean;
-}, z.core.$strip>;
-/**
- * Type representing the validated command options.
- *
- * This type is inferred from the Zod schema and should be used throughout the command
- * to ensure type safety and consistency with the schema.
- */
-export type CommandOptions = z.infer<typeof CommandOptionsSchema>;