@prsense/workflows 0.1.0

package/src/index/indexWorkflow.ts

@@ -0,0 +1,411 @@
+// packages/workflows/src/index/indexWorkflow.ts
+
+import path from "node:path";
+import { CoreEvents, EventBus, ContextChunk } from "@prsense/core";
+import {
+  FileSystemRepositorySource,
+  GitHubRepositorySource,
+  PostgresIndexMetadataRepository,
+  GitLabRepositorySource,
+  PostgresRagChunkRepository,
+  createCharChunker,
+  detectKind,
+  detectLanguage,
+} from "@prsense/context";
+import type { IndexWorkflowResult } from "./types.js";
+import type { ResolvedConfig, CredentialContext } from "@prsense/config";
+import {
+  createOpenAiEmbeddingClient,
+  createOllamaEmbeddingClient,
+} from "@prsense/llm";
+
+//TODO: modularise this
+export async function runIndexWorkflow({
+  config,
+  credentials,
+  target,
+  force,
+  dryRun,
+  eventBus,
+  version,
+}: {
+  config: ResolvedConfig;
+  credentials: CredentialContext;
+  target: string;
+  force?: boolean;
+  dryRun?: boolean;
+  eventBus: EventBus;
+  version: string;
+}): Promise<IndexWorkflowResult> {
+  eventBus.emit(CoreEvents.WorkflowIndexStarted);
+
+  try {
+    // -------------------------------------------------
+    // Resolve Repository Source
+    // -------------------------------------------------
+
+    let repositorySource;
+
+    const isGithub = /github\.com/.test(target);
+    const isGitlab = /gitlab\.com/.test(target);
+
+    if (isGithub) {
+      const match = target.match(/github\.com\/([^\/]+)\/([^\/]+)/);
+
+      if (!match) {
+        throw new Error("Invalid GitHub URL");
+      }
+
+      const owner = match[1];
+      const repo = match[2];
+
+      if (!owner || !repo) {
+        throw new Error("Invalid GitHub repository url");
+      }
+
+      repositorySource = new GitHubRepositorySource(
+        owner,
+        repo.replace(".git", ""),
+      );
+    } else if (isGitlab) {
+      const match = target.match(/gitlab\.com\/(.+?)\/([^\/]+)(?:\.git)?$/);
+
+      if (!match) throw new Error("Invalid GitLab URL");
+
+      const owner = match[1];
+      const repo = match[2];
+      if (!owner || !repo) throw new Error("Invalid GitLab repository url");
+      repositorySource = new GitLabRepositorySource(
+        owner,
+        repo.replace(".git", ""),
+      );
+    } else {
+      const absolute = path.resolve(target);
+      repositorySource = new FileSystemRepositorySource(absolute);
+    }
+
+    // -------------------------------------------------
+    // Resolve Identity + Revision
+    // -------------------------------------------------
+
+    const identity = repositorySource.getRepositoryIdentity();
+    let revision;
+
+    try {
+      revision = await repositorySource.getRevision();
+    } catch {
+      throw new Error(
+        "Indexing requires a git repository. Run inside a git repository.",
+      );
+    }
+
+    // -------------------------------------------------
+    // Metadata Repository
+    // -------------------------------------------------
+
+    const metadataRepository = new PostgresIndexMetadataRepository(
+      config.database.url,
+    );
+
+    const chunkRepository = new PostgresRagChunkRepository(config.database.url);
+
+    const stored = await metadataRepository.load(
+      identity.provider,
+      identity.id,
+    );
+
+    // -------------------------------------------------
+    // Create Embedding Client
+    // -------------------------------------------------
+
+    let embeddingClient;
+
+    if (config.embeddings.provider === "openai") {
+      const apiKey = credentials.openai?.apiKey;
+      if (!apiKey) {
+        throw new Error("OpenAI embedding credentials missing");
+      }
+
+      embeddingClient = createOpenAiEmbeddingClient({
+        apiKey,
+        model: config.embeddings.model,
+      });
+    } else {
+      embeddingClient = createOllamaEmbeddingClient({
+        model: config.embeddings.model,
+      });
+    }
+
+    const embeddingDimension = await embeddingClient.dimension();
+    eventBus.emit(CoreEvents.WorkflowIndexEmbeddingDimensionDetected, {
+      dimension: embeddingDimension,
+    });
+
+    const dbDimension = await chunkRepository.getEmbeddingColumnDimension();
+
+    if (dbDimension !== null && dbDimension !== embeddingDimension) {
+      eventBus.emit(CoreEvents.WorkflowIndexDimensionMismatch, {
+        dbDimension,
+        embeddingDimension,
+      });
+
+      throw new Error(
+        `Embedding dimension mismatch: database=${dbDimension}, model=${embeddingDimension}. Recreate table or change model.`,
+      );
+    }
+
+    // -------------------------------------------------
+    // Compute Current Fingerprint
+    // -------------------------------------------------
+
+    const currentFingerprint = {
+      commitSha: revision.commitSha,
+      embeddingProvider: config.embeddings.provider,
+      embeddingModel: config.embeddings.model,
+      embeddingDimension,
+      chunkStrategy: "default",
+      chunkVersion: 1,
+    };
+
+    let rebuildRequired = false;
+
+    if (!stored) {
+      rebuildRequired = true;
+    } else {
+      if (stored.revision.commitSha !== currentFingerprint.commitSha) {
+        rebuildRequired = true;
+      }
+
+      if (stored.embedding.provider !== currentFingerprint.embeddingProvider) {
+        rebuildRequired = true;
+      }
+
+      if (stored.embedding.model !== currentFingerprint.embeddingModel) {
+        rebuildRequired = true;
+      }
+
+      if (stored.chunking.strategy !== currentFingerprint.chunkStrategy) {
+        rebuildRequired = true;
+      }
+
+      if (stored.chunking.version !== currentFingerprint.chunkVersion) {
+        rebuildRequired = true;
+      }
+    }
+
+    // -------------------------------------------------
+    // Up-to-date Case
+    // -------------------------------------------------
+
+    if (!rebuildRequired) {
+      eventBus.emit(CoreEvents.WorkflowIndexUpToDate, {
+        commitSha: revision.commitSha,
+      });
+
+      eventBus.emit(CoreEvents.WorkflowIndexFinished);
+
+      return {
+        outcome: "success",
+        payload: {
+          chunksIndexed: 0,
+          commitSha: revision.commitSha,
+          upToDate: true,
+        },
+      };
+    }
+
+    // -------------------------------------------------
+    // Outdated Case
+    // -------------------------------------------------
+
+    eventBus.emit(CoreEvents.WorkflowIndexOutdated, {
+      commitSha: revision.commitSha,
+    });
+
+    if (dryRun) {
+      eventBus.emit(CoreEvents.WorkflowIndexFinished);
+      return {
+        outcome: "success",
+        payload: { chunksIndexed: 0 },
+      };
+    }
+
+    if (!force && stored) {
+      eventBus.emit(CoreEvents.WorkflowIndexRebuildRequired, {
+        reason: "index-outdated",
+      });
+
+      eventBus.emit(CoreEvents.WorkflowIndexFinished);
+
+      return {
+        outcome: "success",
+        payload: {
+          chunksIndexed: 0,
+          commitSha: revision.commitSha,
+          upToDate: false,
+        },
+      };
+    }
+
+    // -------------------------------------------------
+    // Rebuild
+    // -------------------------------------------------
+
+    const files = await repositorySource.listFiles();
+    if (files.length === 0) {
+      eventBus.emit(CoreEvents.WorkflowIndexFinished);
+
+      return {
+        outcome: "success",
+        payload: {
+          chunksIndexed: 0,
+          commitSha: revision.commitSha,
+          upToDate: false,
+        },
+      };
+    }
+    const chunker = createCharChunker({
+      maxChars: config.index.chunkSizeChars,
+      overlapChars: config.index.chunkOverlapChars,
+    });
+
+    const chunks: ContextChunk[] = [];
+    for (const file of files) {
+      try {
+        const content = await repositorySource.readFile(file);
+        const kind = detectKind(file);
+        const language = detectLanguage(file);
+
+        const fileChunks = chunker.chunk({
+          content,
+          source: { kind: "file", path: file },
+        });
+
+        for (const chunk of fileChunks) {
+          const metadata: typeof chunk.metadata = {
+            ...chunk.metadata,
+            path: file,
+            kind,
+          };
+          if (language !== undefined) metadata.language = language;
+          chunk.metadata = metadata;
+        }
+
+        chunks.push(...fileChunks);
+      } catch (err) {
+        if (err instanceof Error && err.message === "BINARY_FILE_DETECTED") {
+          eventBus.emit(CoreEvents.ContextFileSkipped, {
+            path: file,
+            reason: "binary",
+          });
+          continue;
+        }
+
+        throw err;
+      }
+    }
+
+    eventBus.emit(CoreEvents.ContextChunksBuilt, {
+      count: chunks.length,
+    });
+
+    // -------------------------------------------------
+    // Delete Existing Chunks (Rebuild)
+    // -------------------------------------------------
+
+    await chunkRepository.deleteByRepository(identity.provider, identity.id);
+
+    // -------------------------------------------------
+    // Embed + Persist Chunks
+    // -------------------------------------------------
+
+    const allRows = [];
+
+    //PERF: different batch size for openai based embedding
+    //PERF: multi threaded?
+    const BATCH_SIZE = 32;
+
+    for (let i = 0; i < chunks.length; i += BATCH_SIZE) {
+      const batch = chunks.slice(i, i + BATCH_SIZE);
+
+      eventBus.emit(CoreEvents.WorkflowIndexProgress, {
+        processed: Math.min(i + BATCH_SIZE, chunks.length),
+        total: chunks.length,
+      });
+
+      const embeddings = await embeddingClient.embed(
+        batch.map((c) => c.content),
+      );
+
+      const rows = batch.map((chunk, idx) => {
+        const embedding = embeddings[idx];
+
+        if (!embedding) {
+          throw new Error("Embedding generation mismatch");
+        }
+        return {
+          chunk,
+          repoProvider: identity.provider,
+          repoName: identity.id,
+          repoRef: revision.commitSha,
+          embedding,
+        };
+      });
+
+      allRows.push(...rows);
+    }
+
+    await chunkRepository.rebuildRepository(
+      identity.provider,
+      identity.id,
+      allRows,
+    );
+    await metadataRepository.save({
+      repository: {
+        provider: identity.provider,
+        id: identity.id,
+        ...(revision.defaultBranch
+          ? { defaultBranch: revision.defaultBranch }
+          : {}),
+      },
+      revision: {
+        commitSha: revision.commitSha,
+      },
+      embedding: {
+        provider: config.embeddings.provider,
+        model: config.embeddings.model,
+        dimension: embeddingDimension,
+      },
+      chunking: {
+        strategy: "default",
+        version: 1,
+      },
+      prsenseVersion: version,
+      createdAt: new Date().toISOString(),
+    });
+
+    eventBus.emit(CoreEvents.WorkflowIndexFinished);
+
+    return {
+      outcome: "success",
+      payload: {
+        chunksIndexed: chunks.length,
+        commitSha: revision.commitSha,
+        upToDate: false,
+      },
+    };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+
+    eventBus.emit(CoreEvents.WorkflowIndexFailed, {
+      error: message,
+    });
+
+    return {
+      outcome: "failure",
+      payload: {
+        chunksIndexed: 0,
+      },
+    };
+  }
+}

package/src/index/listIndexedRepositories.ts

@@ -0,0 +1,11 @@
+import type { ResolvedConfig } from "@prsense/config";
+import { PostgresIndexMetadataRepository } from "@prsense/context";
+import type { IndexedRepository } from "@prsense/core";
+
+export async function listIndexedRepositories(
+  config: ResolvedConfig,
+): Promise<IndexedRepository[]> {
+  const repo = new PostgresIndexMetadataRepository(config.database.url);
+
+  return repo.list();
+}

package/src/index/ports.ts

@@ -0,0 +1,8 @@
+// packages/workflows/src/index/ports.ts
+
+import type { ContextChunk } from "@prsense/core";
+
+export interface ContextIndexer {
+  buildChunks(): Promise<ContextChunk[]>;
+  persistChunks(chunks: ContextChunk[]): Promise<void>;
+}

package/src/index/types.ts

@@ -0,0 +1,11 @@
+// packages/workflows/src/index/types.ts
+
+import { WorkflowResult } from "../types/core.js";
+
+export type IndexPayload = {
+  chunksIndexed: number;
+  commitSha?: string;
+  upToDate?: boolean;
+};
+
+export type IndexWorkflowResult = WorkflowResult<IndexPayload>;

package/src/index.ts

@@ -0,0 +1,13 @@
+export * from "./doctor/workflow.js";
+
+export * from "./index/index.js";
+export * from "./index/indexWorkflow.js";
+
+export * from "./review/types.js";
+
+export * from "./setup/types.js";
+export * from "./setup/setupWorkflow.js";
+
+export * from "./types/core.js";
+
+export * from "./review/index.js";

package/src/review/buildDiffEmbeddingQuery.ts

@@ -0,0 +1,66 @@
+// packages/workflows/src/review/buildDiffEmbeddingQuery
+import type { UnifiedDiff } from "@prsense/core";
+
+export function buildDiffEmbeddingQuery(params: {
+  diff: UnifiedDiff;
+  title?: string;
+  description?: string;
+  maxChars?: number;
+}): string {
+  const { diff, title, description, maxChars = 4000 } = params;
+
+  const parts: string[] = [];
+
+  // -------------------------------------------------
+  // PR Intent
+  // -------------------------------------------------
+
+  if (title) {
+    parts.push(`PR Title: ${title}`);
+  }
+
+  if (description) {
+    parts.push(`PR Description: ${description}`);
+  }
+
+  // -------------------------------------------------
+  // Changed Files (strong retrieval anchor)
+  // -------------------------------------------------
+
+  parts.push("Changed Files:");
+
+  for (const file of diff.files.slice(0, 50)) {
+    parts.push(file.path);
+  }
+
+  // -------------------------------------------------
+  // Meaningful Code Changes
+  // -------------------------------------------------
+
+  parts.push("\nChanged Code:");
+
+  for (const file of diff.files) {
+    parts.push(`File: ${file.path}`);
+
+    const lines = file.patch.split("\n");
+
+    for (const line of lines) {
+      // Only meaningful changes
+      if (line.startsWith("+") && !line.startsWith("+++")) {
+        parts.push(line.slice(1));
+      }
+
+      if (line.startsWith("-") && !line.startsWith("---")) {
+        parts.push(line.slice(1));
+      }
+    }
+  }
+
+  let query = parts.join("\n");
+
+  if (query.length > maxChars) {
+    query = query.slice(0, maxChars);
+  }
+
+  return query;
+}

package/src/review/dedupeSignals.ts

@@ -0,0 +1,10 @@
+import { ReviewSignal } from "@prsense/core";
+export function dedupeSignals(signals: ReviewSignal[]): ReviewSignal[] {
+  const seen = new Set<string>();
+  return signals.filter((s) => {
+    const key = `${s.file}:${s.lineStart}:${s.message}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}

package/src/review/extractJson.ts

@@ -0,0 +1,17 @@
+export function extractJson(text: string): string {
+  // Remove markdown fences if present
+  const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i);
+  if (fenced && fenced[1]) {
+    return fenced[1].trim();
+  }
+
+  // Otherwise attempt brace extraction
+  const first = text.indexOf("{");
+  const last = text.lastIndexOf("}");
+
+  if (first !== -1 && last !== -1) {
+    return text.slice(first, last + 1);
+  }
+
+  return text;
+}

package/src/review/index.ts

@@ -0,0 +1,3 @@
+export * from "./retrieveContext.js";
+export * from "./reviewWorkflow.js";
+export * from "./validateReviewOutput.js";

package/src/review/input/ReviewInput.ts

@@ -0,0 +1,22 @@
+// packages/core/input/ReviewInput.ts
+
+export type ReviewInput = {
+  repo: {
+    owner: string;
+    name: string;
+  };
+
+  /** Raw unified diff text */
+  diffText: string;
+
+  /** Optional hints from adapter */
+  repoRoot?: string;
+  baseBranch?: string;
+
+  /** Human-authored metadata */
+  metadata?: {
+    title?: string;
+    description?: string;
+    author?: string;
+  };
+};

package/src/review/normalizeSignal.ts

@@ -0,0 +1,34 @@
+import type { ReviewSignal } from "@prsense/core";
+
+const allowedTypes = ["bug", "risk", "test", "style"] as const;
+const allowedSeverity = ["low", "medium", "high"] as const;
+
+export function normalizeSignal(raw: any, index: number): ReviewSignal | null {
+  if (!raw || typeof raw !== "object") return null;
+
+  if (!allowedTypes.includes(raw.type)) return null;
+  if (!allowedSeverity.includes(raw.severity)) return null;
+
+  const confidence =
+    typeof raw.confidence === "number"
+      ? Math.max(0, Math.min(1, raw.confidence))
+      : 0.5;
+
+  if (!raw.file || typeof raw.file !== "string") return null;
+  if (!raw.message || typeof raw.message !== "string") return null;
+
+  return {
+    id: `signal-${index}`,
+    type: raw.type,
+    severity: raw.severity,
+    confidence,
+    file: raw.file,
+    lineStart: typeof raw.lineStart === "number" ? raw.lineStart : undefined,
+    lineEnd: typeof raw.lineEnd === "number" ? raw.lineEnd : undefined,
+    message: raw.message,
+    rationale: typeof raw.rationale === "string" ? raw.rationale : undefined,
+    suggestedFix:
+      typeof raw.suggestedFix === "string" ? raw.suggestedFix : undefined,
+    source: "llm",
+  };
+}

package/src/review/ports.ts

@@ -0,0 +1,7 @@
+// packages/workflows/src/review/ports.ts
+
+import type { ContextChunk, UnifiedDiff, ReviewSignal } from "@prsense/core";
+
+export interface ReviewSignalCompiler {
+  compile(diff: UnifiedDiff, context: ContextChunk[]): Promise<ReviewSignal[]>;
+}