@equinor/fusion-framework-cli-plugin-ai-index 1.0.0

package/src/utils/git/file-changes.ts

@@ -0,0 +1,213 @@
+import { join, relative } from 'node:path';
+import type { ChangedFile, FileChangeStatus, GitDiffOptions } from './types.js';
+import { resolveProjectRoot, getGit } from './git-client.js';
+
+/**
+ * Get list of changed files using git diff with status
+ * @param options - Git diff configuration options
+ * @returns Array of changed files with their status
+ */
+export const getChangedFiles = async (options: GitDiffOptions): Promise<ChangedFile[]> => {
+  const { diff, baseRef = 'HEAD~1', cwd = process.cwd() } = options;
+
+  if (!diff) {
+    return [];
+  }
+
+  const projectRoot = resolveProjectRoot(cwd);
+  if (!projectRoot) {
+    throw new Error('Not in a git repository. Cannot use --diff option.');
+  }
+
+  const { git } = getGit(cwd) ?? {};
+  if (!git) {
+    throw new Error('Failed to initialize git client');
+  }
+
+  try {
+    // Get changes since baseRef with status (A=added, M=modified, D=deleted)
+    try {
+      const diffResult = await git.diff([`${baseRef}`, '--name-status']);
+      const lines = diffResult.split('\n').filter((line) => line.trim() !== '');
+
+      const changedFiles: ChangedFile[] = [];
+
+      for (const line of lines) {
+        // Match status and file path
+        // Format: "A\tfile.ts" or "M\tfile.ts" or "D\tfile.ts"
+        // Also handle renames: "R100\told.ts\tnew.ts"
+        const renameMatch = line.match(/^R\d*\s+(.+?)\s+(.+)$/);
+        if (renameMatch) {
+          const [, oldFile, newFile] = renameMatch;
+          // Add both the removed old file and the new file
+          changedFiles.push({ filepath: `${projectRoot}/${oldFile}`, status: 'removed' });
+          changedFiles.push({ filepath: `${projectRoot}/${newFile}`, status: 'new' });
+          continue;
+        }
+
+        const match = line.match(/^([AMD])\s+(.+)$/);
+        if (match) {
+          const [, gitStatus, file] = match;
+          const fullPath = `${projectRoot}/${file}`;
+
+          let status: FileChangeStatus;
+          if (gitStatus === 'A') {
+            status = 'new';
+          } else if (gitStatus === 'M') {
+            status = 'modified';
+          } else if (gitStatus === 'D') {
+            status = 'removed';
+          } else {
+            // Skip unknown statuses (C=copied, etc.)
+            continue;
+          }
+
+          changedFiles.push({ filepath: fullPath, status });
+        }
+      }
+
+      return changedFiles;
+    } catch {
+      // Handle case where baseRef doesn't exist (e.g., first commit)
+      console.warn(`⚠️  Warning: Git reference '${baseRef}' not found. Processing all files.`);
+      return [];
+    }
+  } catch (error) {
+    throw new Error(`Git diff failed: ${error instanceof Error ? error.message : String(error)}`);
+  }
+};
+
+/**
+ * Determine the git status of a file, including handling renames
+ * Returns an array of ChangedFile objects - if the file was renamed, returns both old and new paths
+ * @param filePath - Absolute file path to check
+ * @returns Promise resolving to array of changed files (1 or 2 items if renamed)
+ */
+export const getFileStatus = async (filePath: string): Promise<ChangedFile[]> => {
+  const { git, gitRepoPath } = getGit(filePath) ?? {};
+  if (!git || !gitRepoPath) {
+    // Not in a git repository, assume new
+    return [{ filepath: filePath, status: 'new' }];
+  }
+
+  const gitFilePath = relative(gitRepoPath, filePath);
+  // Normalize path separators for git commands (git uses forward slashes on all platforms)
+  const normalizedGitFilePath = gitFilePath.replace(/\\/g, '/');
+
+  try {
+    // First check if file is tracked in git at the current path
+    const isTracked = await git
+      .raw(['ls-files', '--error-unmatch', normalizedGitFilePath])
+      .then(() => true)
+      .catch(() => false);
+
+    if (isTracked) {
+      // File is tracked at this path, it's modified
+      return [{ filepath: filePath, status: 'modified' }];
+    }
+
+    // File is not tracked - quickly check if it's explicitly untracked
+    // This is much faster than checking full status or history
+    try {
+      const fileStatusOutput = await git.raw([
+        'status',
+        '--porcelain',
+        '--',
+        normalizedGitFilePath,
+      ]);
+      const trimmed = fileStatusOutput.trim();
+
+      if (trimmed.length > 0) {
+        // If status shows ??, it's untracked (truly new)
+        if (/^\?\?/.test(trimmed)) {
+          return [{ filepath: filePath, status: 'new' }];
+        }
+      }
+    } catch {
+      // If status check fails, continue to rename/history checks
+    }
+
+    // File is not tracked and not explicitly untracked - check if it's a rename
+    // Only do expensive checks if we haven't determined status yet
+    try {
+      // Get full git status to check for renames (only if needed)
+      const statusOutput = await git.raw(['status', '--porcelain']);
+      const lines = statusOutput.split('\n').filter((line) => line.trim() !== '');
+
+      for (const line of lines) {
+        // Check for rename format: "R100\told.ts\tnew.ts"
+        const renameMatch = line.match(/^R\d+\s+(.+?)\s+(.+)$/);
+        if (renameMatch) {
+          const [, oldPath, newPath] = renameMatch;
+          const oldFullPath = join(gitRepoPath, oldPath);
+          const newFullPath = join(gitRepoPath, newPath);
+
+          // Check if the current file is the new path in a rename
+          if (newFullPath === filePath) {
+            return [
+              { filepath: oldFullPath, status: 'removed' },
+              { filepath: newFullPath, status: 'new' },
+            ];
+          }
+        }
+
+        // Check for copy format: "C100\told.ts\tnew.ts" (similar to rename)
+        const copyMatch = line.match(/^C\d+\s+(.+?)\s+(.+)$/);
+        if (copyMatch) {
+          const [, , newPath] = copyMatch;
+          const newFullPath = join(gitRepoPath, newPath);
+
+          // For copies, the old file still exists, so only return the new one
+          if (newFullPath === filePath) {
+            return [{ filepath: newFullPath, status: 'new' }];
+          }
+        }
+      }
+    } catch {
+      // If status check fails, continue to history check
+    }
+
+    // Last resort: check if file content exists in git history (very slow, only if needed)
+    // Use --follow to track renames, limit to 1 commit for performance
+    try {
+      const hasHistory = await git
+        .raw([
+          'log',
+          '--all',
+          '--full-history',
+          '--follow',
+          '--oneline',
+          '-1',
+          '--',
+          normalizedGitFilePath,
+        ])
+        .then((output) => output.trim().length > 0)
+        .catch(() => false);
+
+      // If file has history but isn't tracked, it might have been moved
+      // For now, treat as 'new' at the new location
+      // Note: We can't easily find the old path without more complex git operations
+      return [{ filepath: filePath, status: hasHistory ? 'modified' : 'new' }];
+    } catch {
+      // If we can't determine, default to 'new'
+      return [{ filepath: filePath, status: 'new' }];
+    }
+  } catch {
+    // If we can't determine status, default to 'new'
+    return [{ filepath: filePath, status: 'new' }];
+  }
+};
+
+/**
+ * Check if a file path matches any of the changed files
+ * @param filePath - File path to check
+ * @param changedFiles - Array of changed file objects
+ * @returns True if file has changed
+ */
+export const isFileChanged = (filePath: string, changedFiles: ChangedFile[]): boolean => {
+  if (changedFiles.length === 0) {
+    return true; // If no diff filtering, process all files
+  }
+
+  return changedFiles.some((file) => file.filepath === filePath);
+};

package/src/utils/git/git-client.ts

@@ -0,0 +1,43 @@
+import { simpleGit, type SimpleGit } from 'simple-git';
+import { findUpSync } from 'find-up';
+import { dirname, join } from 'node:path';
+import { existsSync } from 'node:fs';
+
+const gitCache = new Map<string, SimpleGit>();
+
+/**
+ * Resolve the project root (git repository root) for a given file path
+ * @param filePath - File path to resolve from
+ * @returns Project root path or undefined if not in a git repository
+ */
+export const resolveProjectRoot = (filePath: string): string | undefined => {
+  // if we are in the root of the git repository, return the root
+  if (existsSync(join(filePath, '.git'))) {
+    return filePath;
+  }
+  const gitRepoPath = findUpSync('.git', { cwd: dirname(filePath), type: 'both' });
+  const projectRoot = gitRepoPath?.replace(/\.git$/, '');
+  return projectRoot;
+};
+
+/**
+ * Get or create a SimpleGit instance for a given file path
+ * Uses caching to avoid creating multiple instances for the same repository
+ * @param filePath - File path to get git instance for
+ * @returns Git instance and repository path, or undefined if not in a git repository
+ */
+export const getGit = (
+  filePath: string,
+): { git: SimpleGit | undefined; gitRepoPath: string } | undefined => {
+  const gitRepoPath = resolveProjectRoot(filePath);
+  if (gitRepoPath) {
+    if (!gitCache.has(gitRepoPath)) {
+      gitCache.set(gitRepoPath, simpleGit(gitRepoPath));
+    }
+    return {
+      git: gitCache.get(gitRepoPath),
+      gitRepoPath,
+    };
+  }
+  return undefined;
+};

package/src/utils/git/index.ts

@@ -0,0 +1,19 @@
+// Re-export all types
+export type {
+  GitMetadata,
+  GitDiffOptions,
+  FileChangeStatus,
+  ChangedFile,
+} from './types.js';
+
+// Re-export git client utilities
+export { resolveProjectRoot, getGit } from './git-client.js';
+
+// Re-export metadata functions
+export { extractGitMetadata } from './metadata.js';
+
+// Re-export file change functions
+export { getChangedFiles, getFileStatus, isFileChanged } from './file-changes.js';
+
+// Re-export status functions
+export { getGitStatus } from './status.js';

package/src/utils/git/metadata.ts

@@ -0,0 +1,47 @@
+import { relative } from 'node:path';
+import type { GitMetadata } from './types.js';
+import { getGit } from './git-client.js';
+
+/**
+ * Generate a GitHub permalink for a file
+ * @param gitRemoteUrl - Git remote URL
+ * @param filePath - Relative file path from repository root
+ * @param slug - Git reference (branch/tag/commit), defaults to 'main'
+ * @returns GitHub permalink URL or undefined if not a GitHub repository
+ * @internal
+ */
+const generateGithubPermalink = (
+  gitRemoteUrl: string,
+  filePath: string,
+  slug?: string,
+): string | undefined => {
+  const githubMatch = gitRemoteUrl.match(/github\.com[:/]([^/]+)\/([^/.]+)(?:\.git)?$/);
+  if (githubMatch) {
+    const [, owner, repo] = githubMatch;
+    return `https://github.com/${owner}/${repo}/blob/${slug ?? 'main'}/${filePath}`;
+  }
+  return undefined;
+};
+
+/**
+ * Extract git metadata for a file
+ * @param filePath - Absolute file path
+ * @returns Git metadata or undefined if not in a git repository
+ */
+export const extractGitMetadata = async (filePath: string): Promise<GitMetadata | undefined> => {
+  const { git, gitRepoPath: gitRepoRoot } = getGit(filePath) ?? {};
+  if (!git || !gitRepoRoot) {
+    return undefined;
+  }
+  const gitFilePath = relative(gitRepoRoot, filePath);
+  const { latest } = await git.log({ file: gitFilePath, maxCount: 1 });
+  const gitRemoteUrl = await git
+    .getConfig('remote.origin.url')
+    .then(({ value }) => value ?? undefined);
+  const git_link = gitRemoteUrl ? generateGithubPermalink(gitRemoteUrl, gitFilePath) : undefined;
+  return {
+    git_link,
+    git_commit_hash: latest?.hash,
+    git_commit_date: latest?.date,
+  };
+};

package/src/utils/git/status.ts

@@ -0,0 +1,48 @@
+import { resolveProjectRoot, getGit } from './git-client.js';
+
+/**
+ * Get git status information for debugging
+ * @param cwd - Working directory
+ * @returns Git status information
+ */
+export const getGitStatus = async (
+  cwd: string = process.cwd(),
+): Promise<{
+  branch: string;
+  commit: string;
+  hasChanges: boolean;
+  stagedFiles: number;
+  unstagedFiles: number;
+}> => {
+  const projectRoot = resolveProjectRoot(cwd);
+  if (!projectRoot) {
+    throw new Error('Not in a git repository');
+  }
+
+  const { git } = getGit(cwd) ?? {};
+  if (!git) {
+    throw new Error('Failed to initialize git client');
+  }
+
+  try {
+    const branch = await git.revparse(['--abbrev-ref', 'HEAD']);
+    const commit = await git.revparse(['--short', 'HEAD']);
+
+    const statusResult = await git.status();
+    const stagedFiles = statusResult.staged.length;
+    const unstagedFiles =
+      statusResult.modified.length + statusResult.deleted.length + statusResult.not_added.length;
+
+    return {
+      branch: branch.trim(),
+      commit: commit.trim(),
+      hasChanges: stagedFiles > 0 || unstagedFiles > 0,
+      stagedFiles,
+      unstagedFiles,
+    };
+  } catch (error) {
+    throw new Error(
+      `Failed to get git status: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+};

package/src/utils/git/types.ts

@@ -0,0 +1,36 @@
+/**
+ * Git metadata extracted from repository
+ */
+export type GitMetadata = Partial<{
+  git_remote_url: string;
+  git_commit_hash: string;
+  git_commit_date: string;
+  git_link: string;
+}>;
+
+/**
+ * Git diff options for filtering changed files
+ */
+export interface GitDiffOptions {
+  /** Enable diff-based file filtering */
+  diff: boolean;
+  /** Git reference to compare against (default: HEAD~1) */
+  baseRef?: string;
+  /** Working directory for git operations */
+  cwd?: string;
+}
+
+/**
+ * File change status
+ */
+export type FileChangeStatus = 'new' | 'modified' | 'removed';
+
+/**
+ * Changed file information
+ */
+export interface ChangedFile {
+  /** Absolute file path */
+  filepath: string;
+  /** Change status: new, modified, or removed */
+  status: FileChangeStatus;
+}

package/src/utils/markdown/index.ts

@@ -0,0 +1,5 @@
+// Re-export all types
+export type { MarkdownMetadata, MarkdownDocument } from './types.js';
+
+// Re-export parser functions
+export { isMarkdownFile, parseMarkdown, parseMarkdownFile } from './parser.js';

package/src/utils/markdown/parser.ts

@@ -0,0 +1,92 @@
+import { readFileSync } from 'node:fs';
+import { assert } from 'node:console';
+
+import { default as grayMatter } from 'gray-matter';
+
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
+
+import type { SourceFile } from '../types.js';
+import type { MarkdownDocument, MarkdownMetadata } from './types.js';
+import { generateChunkId } from '../generate-chunk-id.js';
+
+const markdownConfig = {
+  chunkSize: 3000,
+  chunkOverlap: 300,
+  keepSeparator: true,
+  separators: [
+    '\n# ',
+    '\n## ',
+    '\n### ',
+    '\n#### ',
+    '\n##### ',
+    '\n###### ',
+    '\n```',
+    '\n---\n',
+    '\n\n',
+  ],
+};
+
+/**
+ * Check if a file is a markdown or MDX file
+ * @param filePath - File path to check
+ * @returns True if file has .md or .mdx extension
+ */
+export const isMarkdownFile = (filePath: string): boolean => {
+  return filePath.endsWith('.md') || filePath.endsWith('.mdx');
+};
+
+/**
+ * Parse markdown or MDX content into document chunks
+ * @param content - Markdown or MDX content string
+ * @param source - Source file path
+ * @returns Array of markdown documents
+ */
+export const parseMarkdown = async <T extends Record<string, unknown> = Record<string, unknown>>(
+  content: string,
+  source: string,
+): Promise<MarkdownDocument<T>[]> => {
+  const { content: markdownContent, data } = grayMatter(content);
+  const markdownAttributes = Object.entries(data).reduce(
+    (acc, [key, value]) => {
+      acc[`md_${key}`] = value;
+      return acc;
+    },
+    {
+      type: 'markdown',
+    } as Record<string, unknown>,
+  );
+  const textSplitter = new RecursiveCharacterTextSplitter(markdownConfig);
+  const chunks = await textSplitter.splitText(markdownContent);
+  return chunks.map(
+    (chunk, _index): MarkdownDocument<T> => ({
+      id: generateChunkId(source, _index),
+      pageContent: chunk,
+      metadata: {
+        source,
+        attributes: markdownAttributes as MarkdownMetadata<T>['attributes'],
+      },
+    }),
+  );
+};
+
+/**
+ * Parse a markdown or MDX file into document chunks
+ * @param file - Source file object
+ * @returns Array of markdown documents with root path metadata
+ */
+export const parseMarkdownFile = async <
+  T extends Record<string, unknown> = Record<string, unknown>,
+>(
+  file: SourceFile,
+): Promise<MarkdownDocument<T>[]> => {
+  assert(isMarkdownFile(file.path), `File ${file.path} is not a markdown or MDX file`);
+  const content = readFileSync(file.path, 'utf8');
+  const result = await parseMarkdown<T>(content, file.relativePath ?? file.path);
+  return result.map((document) => ({
+    ...document,
+    metadata: {
+      ...document.metadata,
+      rootPath: file.projectRoot,
+    },
+  }));
+};

package/src/utils/markdown/types.ts

@@ -0,0 +1,20 @@
+import type {
+  VectorStoreDocument,
+  VectorStoreDocumentMetadata,
+} from '@equinor/fusion-framework-module-ai/lib';
+
+/**
+ * Markdown document metadata
+ */
+export type MarkdownMetadata<T extends Record<string, unknown> = Record<string, unknown>> =
+  VectorStoreDocumentMetadata<
+    T & {
+      type: 'markdown';
+    }
+  >;
+
+/**
+ * Markdown document
+ */
+export type MarkdownDocument<T extends Record<string, unknown> = Record<string, unknown>> =
+  VectorStoreDocument<MarkdownMetadata<T>>;

package/src/utils/package-resolver.ts

@@ -0,0 +1,44 @@
+import { dirname } from 'node:path';
+import { readPackageUp, type PackageJson } from 'read-package-up';
+
+/**
+ * Map of package base directories to package.json
+ */
+const packageMap = new Map<string, PackageJson>();
+
+/**
+ * Resolves which package a file path belongs to.
+ * First checks the cache map, then uses read-package-up if no match found.
+ *
+ * @param filePath - Absolute or relative file path (e.g., '/path/to/packages/cli/src/index.ts')
+ * @returns Package.json if found, undefined otherwise
+ *
+ * @example
+ * ```ts
+ * const packageJson = await resolvePackage('/path/to/packages/cli/src/index.ts');
+ * ```
+ */
+export async function resolvePackage(filePath: string): Promise<PackageJson | undefined> {
+  // Check cache: iterate through known package directories
+  for (const packageRoot of packageMap.keys()) {
+    if (filePath.startsWith(packageRoot)) {
+      const packageJson = packageMap.get(packageRoot);
+      if (packageJson) {
+        return packageJson;
+      }
+    }
+  }
+
+  // Not in cache, resolve using read-package-up
+  // readPackageUp expects a directory path, not a file path
+  const dirPath = dirname(filePath);
+  const result = await readPackageUp({ cwd: dirPath, normalize: false });
+
+  if (result) {
+    // Cache using the package directory (where package.json is located)
+    const packageDir = dirname(result.path);
+    packageMap.set(packageDir, result.packageJson);
+  }
+
+  return result?.packageJson;
+}

package/src/utils/ts-doc/constants.ts

@@ -0,0 +1,13 @@
+import { SyntaxKind } from 'ts-morph';
+
+/**
+ * Supported TSDoc node kinds for top-level processing
+ */
+export const nodeKinds = [
+  SyntaxKind.FunctionDeclaration,
+  SyntaxKind.ClassDeclaration,
+  SyntaxKind.InterfaceDeclaration,
+  SyntaxKind.TypeAliasDeclaration,
+  SyntaxKind.VariableStatement,
+  SyntaxKind.EnumDeclaration,
+];