@lota-sdk/core 0.1.5

package/src/workers/skill-extraction.runner.ts

@@ -0,0 +1,331 @@
+import { toTimestamp } from '@lota-sdk/shared/runtime/chat-message-metadata'
+import { BoundQuery } from 'surrealdb'
+import { z } from 'zod'
+
+import { serverLogger } from '../config/logger'
+import { ensureRecordId, recordIdToString } from '../db/record-id'
+import type { RecordIdRef } from '../db/record-id'
+import { databaseService } from '../db/service'
+import { TABLES } from '../db/tables'
+import { createDefaultEmbeddings } from '../embeddings/provider'
+import type { SkillExtractionJob } from '../queues/skill-extraction.queue'
+import { createHelperModelRuntime } from '../runtime/helper-model'
+import { getRuntimeAdapters, withConfiguredWorkspaceMemoryLock } from '../runtime/runtime-extensions'
+import { learnedSkillService } from '../services/learned-skill.service'
+import { createSkillExtractorAgent, SkillExtractionOutputSchema } from '../system-agents/skill-extractor.agent'
+import type { SkillCandidate } from '../system-agents/skill-extractor.agent'
+import { createSkillManagerAgent, SkillManagerOutputSchema } from '../system-agents/skill-manager.agent'
+import { buildDigestTranscript, resolveWorkspaceBootstrapCutoff } from './regular-chat-memory-digest.helpers'
+
+const SKILL_EXTRACTION_TIMEOUT_MS = 10 * 60 * 1000
+const MIN_MESSAGE_THRESHOLD = 10
+
+const RecordTimestampSchema = z.union([z.date(), z.string(), z.number()])
+const MessageRoleSchema = z.enum(['system', 'user', 'assistant'])
+const MessagePartSchema = z.record(z.string(), z.unknown())
+const MessageMetadataSchema = z.record(z.string(), z.unknown()).nullish()
+
+const WorkstreamMessageRowSchema = z.object({
+  id: z.string(),
+  workstreamId: z.string(),
+  role: MessageRoleSchema,
+  parts: z.array(MessagePartSchema).optional(),
+  metadata: MessageMetadataSchema,
+  createdAt: RecordTimestampSchema,
+})
+
+interface DigestCursor {
+  createdAt: Date
+  id: string
+}
+
+interface DigestMessage {
+  source: 'workstream'
+  sourceId: string
+  role: 'system' | 'user' | 'assistant'
+  parts: Array<Record<string, unknown>>
+  metadata?: Record<string, unknown>
+  cursor: DigestCursor
+}
+
+interface SkillExtractionRunResult {
+  skipped: boolean
+  processedMessages: number
+  extractedSkills: number
+}
+
+const embeddings = createDefaultEmbeddings()
+
+const helperModelRuntime = createHelperModelRuntime()
+
+function mapWorkstreamRow(row: z.infer<typeof WorkstreamMessageRowSchema>): DigestMessage {
+  return {
+    source: 'workstream',
+    sourceId: row.workstreamId,
+    role: row.role,
+    parts: row.parts ?? [],
+    metadata: row.metadata ?? undefined,
+    cursor: { createdAt: new Date(toTimestamp(row.createdAt)), id: row.id },
+  }
+}
+
+function compareMessageOrder(left: DigestMessage, right: DigestMessage): number {
+  const timeDiff = left.cursor.createdAt.getTime() - right.cursor.createdAt.getTime()
+  if (timeDiff !== 0) return timeDiff
+  return left.cursor.id.localeCompare(right.cursor.id)
+}
+
+async function listWorkstreamIdsForOrg(orgRef: RecordIdRef): Promise<RecordIdRef[]> {
+  const EntityIdRowSchema = z.string().trim().min(1)
+  const ids = await databaseService.query<unknown>(
+    new BoundQuery(
+      `SELECT VALUE type::string(id) FROM ${TABLES.WORKSTREAM}
+       WHERE organizationId = $organizationId`,
+      { organizationId: orgRef },
+    ),
+  )
+  return ids.map((value) => ensureRecordId(EntityIdRowSchema.parse(value), TABLES.WORKSTREAM))
+}
+
+async function listEligibleMessages(params: {
+  workstreamIds: RecordIdRef[]
+  cursor: DigestCursor | null
+  onboardingCutoff: Date | null
+}): Promise<DigestMessage[]> {
+  if (params.workstreamIds.length === 0) return []
+
+  let query: BoundQuery | null = null
+  if (params.cursor) {
+    const cursorRowId = ensureRecordId(params.cursor.id, TABLES.WORKSTREAM_MESSAGE)
+    query = new BoundQuery(
+      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
+       WHERE workstreamId IN $workstreamIds
+         AND (
+           createdAt > $cursorCreatedAt
+           OR (createdAt = $cursorCreatedAt AND id > $cursorRowId)
+         )
+       ORDER BY createdAt ASC, id ASC`,
+      { workstreamIds: params.workstreamIds, cursorCreatedAt: params.cursor.createdAt, cursorRowId },
+    )
+  } else if (params.onboardingCutoff) {
+    query = new BoundQuery(
+      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
+       WHERE workstreamId IN $workstreamIds
+         AND createdAt > $onboardingCutoff
+       ORDER BY createdAt ASC, id ASC`,
+      { workstreamIds: params.workstreamIds, onboardingCutoff: params.onboardingCutoff },
+    )
+  }
+
+  if (!query) return []
+
+  const rows = await databaseService.query<unknown>(query)
+  return rows.map((row) => mapWorkstreamRow(WorkstreamMessageRowSchema.parse(row)))
+}
+
+function buildExtractionPrompt(params: { workspaceName: string; transcript: string; existingSkills: string }): string {
+  return [
+    `Workspace name: ${params.workspaceName}`,
+    '',
+    'Existing learned skills:',
+    params.existingSkills || 'No existing learned skills.',
+    '',
+    'Recent conversation transcript:',
+    params.transcript || 'No transcript.',
+  ].join('\n')
+}
+
+function buildManagerPrompt(params: {
+  candidate: SkillCandidate
+  existingSkill: { name: string; description: string; instructions: string; version: number } | null
+}): string {
+  const parts = [
+    'Candidate skill:',
+    `Name: ${params.candidate.name}`,
+    `Description: ${params.candidate.description}`,
+    `Instructions: ${params.candidate.instructions}`,
+    `Triggers: ${params.candidate.triggers.join(', ')}`,
+    `Confidence: ${params.candidate.confidence}`,
+  ]
+
+  if (params.existingSkill) {
+    parts.push(
+      '',
+      'Most similar existing skill:',
+      `Name: ${params.existingSkill.name}`,
+      `Description: ${params.existingSkill.description}`,
+      `Instructions: ${params.existingSkill.instructions}`,
+      `Version: ${params.existingSkill.version}`,
+    )
+  } else {
+    parts.push('', 'No similar existing skill found.')
+  }
+
+  return parts.join('\n')
+}
+
+export async function runSkillExtraction(data: SkillExtractionJob): Promise<SkillExtractionRunResult> {
+  const orgRef = ensureRecordId(data.orgId, TABLES.ORGANIZATION)
+  const orgId = recordIdToString(orgRef, TABLES.ORGANIZATION)
+  const workspaceProvider = getRuntimeAdapters().services?.workspaceProvider
+  const cursorAwareWorkspaceProvider =
+    workspaceProvider?.getBackgroundCursor && workspaceProvider.setBackgroundCursor
+      ? (workspaceProvider as typeof workspaceProvider & {
+          getBackgroundCursor: NonNullable<typeof workspaceProvider.getBackgroundCursor>
+          setBackgroundCursor: NonNullable<typeof workspaceProvider.setBackgroundCursor>
+        })
+      : undefined
+  if (!cursorAwareWorkspaceProvider) {
+    serverLogger.info`Skipping skill extraction for ${orgId}: workspaceProvider background cursor methods are not configured`
+    return { skipped: true, processedMessages: 0, extractedSkills: 0 }
+  }
+
+  return await withConfiguredWorkspaceMemoryLock(orgId, async () => {
+    const workspace = await cursorAwareWorkspaceProvider.getWorkspace(orgRef)
+    const lifecycleState = await cursorAwareWorkspaceProvider.getLifecycleState?.(workspace)
+    if (lifecycleState?.bootstrapActive ?? false) {
+      serverLogger.info`Skipping skill extraction for ${orgId}: onboarding is not completed`
+      return { skipped: true, processedMessages: 0, extractedSkills: 0 }
+    }
+    const projectionState = await cursorAwareWorkspaceProvider.readProfileProjectionState?.(workspace)
+
+    const existingCursor = await cursorAwareWorkspaceProvider.getBackgroundCursor('skill-extraction', orgRef)
+    const onboardingCutoff = resolveWorkspaceBootstrapCutoff({
+      hasExistingCursor: existingCursor !== null,
+      bootstrapCompletedAt: lifecycleState?.bootstrapCompletedAt,
+    })
+
+    const workstreamIds = await listWorkstreamIdsForOrg(orgRef)
+    const messages = await listEligibleMessages({ workstreamIds, cursor: existingCursor, onboardingCutoff })
+
+    if (messages.length < MIN_MESSAGE_THRESHOLD) {
+      serverLogger.info`Skipping skill extraction for ${orgId}: only ${messages.length} messages (threshold: ${MIN_MESSAGE_THRESHOLD})`
+      return { skipped: true, processedMessages: messages.length, extractedSkills: 0 }
+    }
+
+    const sortedMessages = [...messages].sort(compareMessageOrder)
+    const { transcript } = buildDigestTranscript({ messages: sortedMessages })
+
+    const existingSkills = await learnedSkillService.listForOrg(orgId)
+    const existingSkillsSummary =
+      existingSkills.length > 0
+        ? existingSkills.map((skill, i) => `${i + 1}. ${skill.name}: ${skill.description}`).join('\n')
+        : 'None'
+
+    const extraction = await helperModelRuntime.generateHelperStructured({
+      tag: 'skill-extraction',
+      createAgent: createSkillExtractorAgent,
+      timeoutMs: SKILL_EXTRACTION_TIMEOUT_MS,
+      messages: [
+        {
+          role: 'user',
+          content: buildExtractionPrompt({
+            workspaceName: projectionState?.workspaceName || 'Workspace',
+            transcript,
+            existingSkills: existingSkillsSummary,
+          }),
+        },
+      ],
+      schema: SkillExtractionOutputSchema,
+    })
+
+    const skillCandidates = extraction.candidates.filter((c) => c.classification === 'skill')
+    let extractedSkills = 0
+
+    for (const candidate of skillCandidates) {
+      try {
+        const hash = learnedSkillService.generateHash(candidate.description, candidate.instructions)
+        const existingByHash = await learnedSkillService.findByHash(orgId, hash)
+        if (existingByHash) {
+          serverLogger.info`Skipping duplicate skill candidate ${candidate.name} (hash match)`
+          continue
+        }
+
+        const mostSimilar = await learnedSkillService.findMostSimilar(orgId, candidate.description)
+
+        const managerResult = await helperModelRuntime.generateHelperStructured({
+          tag: 'skill-manager',
+          createAgent: createSkillManagerAgent,
+          timeoutMs: SKILL_EXTRACTION_TIMEOUT_MS,
+          messages: [
+            {
+              role: 'user',
+              content: buildManagerPrompt({
+                candidate,
+                existingSkill: mostSimilar
+                  ? {
+                      name: mostSimilar.name,
+                      description: mostSimilar.description,
+                      instructions: mostSimilar.instructions,
+                      version: mostSimilar.version,
+                    }
+                  : null,
+              }),
+            },
+          ],
+          schema: SkillManagerOutputSchema,
+        })
+
+        if (managerResult.decision === 'discard') {
+          serverLogger.info`Discarding skill candidate ${candidate.name}: ${managerResult.reason}`
+          continue
+        }
+
+        const embedding = await embeddings.embedQuery(candidate.description)
+        if (embedding.length === 0) {
+          serverLogger.warn`Skipping skill candidate ${candidate.name}: empty embedding`
+          continue
+        }
+
+        if (managerResult.decision === 'add') {
+          await learnedSkillService.create({
+            name: candidate.name,
+            description: candidate.description,
+            instructions: candidate.instructions,
+            triggers: candidate.triggers,
+            tags: candidate.tags,
+            examples: candidate.examples,
+            sourceType: 'conversation',
+            organizationId: orgId,
+            agentId: candidate.agentId,
+            confidence: candidate.confidence,
+            embedding,
+            hash,
+          })
+          extractedSkills++
+          serverLogger.info`Added new learned skill: ${candidate.name}`
+        } else if (mostSimilar && managerResult.mergedSkill) {
+          const merged = managerResult.mergedSkill
+          const mergedHash = learnedSkillService.generateHash(merged.description, merged.instructions)
+          const mergedEmbedding = await embeddings.embedQuery(merged.description)
+
+          await learnedSkillService.update(mostSimilar.id, {
+            name: merged.name,
+            description: merged.description,
+            instructions: merged.instructions,
+            triggers: merged.triggers,
+            tags: merged.tags,
+            examples: merged.examples,
+            confidence: merged.confidence,
+            version: mostSimilar.version + 1,
+            embedding: mergedEmbedding,
+            hash: mergedHash,
+          })
+          extractedSkills++
+          serverLogger.info`Merged skill candidate into ${mostSimilar.name} (v${mostSimilar.version + 1})`
+        }
+      } catch (candidateError) {
+        serverLogger.warn`Failed to process skill candidate ${candidate.name}: ${candidateError}`
+      }
+    }
+
+    const lastMessage = sortedMessages.at(-1)
+    if (lastMessage) {
+      await cursorAwareWorkspaceProvider.setBackgroundCursor('skill-extraction', orgRef, lastMessage.cursor)
+    }
+
+    serverLogger.info`Skill extraction completed for ${orgId}: messages=${messages.length}, extracted=${extractedSkills}`
+
+    return { skipped: false, processedMessages: messages.length, extractedSkills }
+  })
+}

package/src/workers/skill-extraction.worker.ts

@@ -0,0 +1,22 @@
+import type { SandboxedJob } from 'bullmq'
+
+import { serverLogger } from '../config/logger'
+import type { SkillExtractionJob } from '../queues/skill-extraction.queue'
+import { initializeSandboxedWorkerRuntime } from './bootstrap'
+import { runSkillExtraction } from './skill-extraction.runner'
+import { toSandboxedWorkerError } from './utils/sandbox-error'
+import { createTracedWorkerProcessor } from './worker-utils'
+
+await initializeSandboxedWorkerRuntime()
+
+const handler = async (job: SandboxedJob<SkillExtractionJob>) => {
+  try {
+    await runSkillExtraction(job.data)
+  } catch (error) {
+    const serialized = toSandboxedWorkerError(error, 'Skill extraction failed')
+    serverLogger.error`${serialized.message}`
+    throw serialized
+  }
+}
+
+export default createTracedWorkerProcessor('skill-extraction', handler)

package/src/workers/utils/repo-indexer-chunker.ts

@@ -0,0 +1,331 @@
+export const DEFAULT_REPOMIX_CHUNK_MAX_CHARS = 250_000
+const MIN_REPOMIX_CHUNK_MAX_CHARS = 4_000
+export const DEFAULT_REPOMIX_CHUNK_MIN_CHARS = 10_000
+const SECTION_SEPARATOR_LENGTH = 2
+const FILE_SECTION_HEADER_SOURCE = '^## File:\\s+(.+)$'
+
+interface RepomixSection {
+  kind: 'preamble' | 'file'
+  content: string
+  filePath?: string
+}
+
+export interface RepomixContextChunk {
+  index: number
+  totalChunks: number
+  content: string
+  charLength: number
+  tokenEstimate: number
+  sectionCount: number
+  fileCount: number
+  firstFilePath: string | null
+  lastFilePath: string | null
+}
+
+interface RepomixChunkOptions {
+  maxChars?: number
+  minChunkChars?: number
+  preserveCodeFenceIntegrity?: boolean
+}
+
+function estimateTokenCountFromChars(text: string): number {
+  if (!text) return 0
+  return Math.ceil(text.length / 3)
+}
+
+function normalizeMaxChars(value?: number): number {
+  if (typeof value !== 'number' || !Number.isFinite(value)) {
+    return DEFAULT_REPOMIX_CHUNK_MAX_CHARS
+  }
+  return Math.max(MIN_REPOMIX_CHUNK_MAX_CHARS, Math.floor(value))
+}
+
+function normalizeMinChunkChars(value: number | undefined, maxChars: number): number {
+  if (typeof value !== 'number' || !Number.isFinite(value)) {
+    return Math.min(DEFAULT_REPOMIX_CHUNK_MIN_CHARS, Math.floor(maxChars * 0.35))
+  }
+  const normalized = Math.max(512, Math.floor(value))
+  return Math.min(normalized, Math.floor(maxChars * 0.6))
+}
+
+async function splitTextByCharBudget(text: string, maxChars: number): Promise<string[]> {
+  const source = text.trim()
+  if (!source) return []
+  if (source.length <= maxChars) return [source]
+
+  const chunks: string[] = []
+  let cursor = 0
+
+  while (cursor < source.length) {
+    let end = Math.min(source.length, cursor + maxChars)
+
+    if (end < source.length) {
+      const breakCandidates = [
+        source.lastIndexOf('\n## File: ', end),
+        source.lastIndexOf('\n```\n', end),
+        source.lastIndexOf('\n\n', end),
+        source.lastIndexOf('\n', end),
+        source.lastIndexOf(' ', end),
+      ]
+      const preferred = breakCandidates.find((position) => position > cursor + Math.floor(maxChars * 0.35))
+      if (typeof preferred === 'number' && preferred > cursor) {
+        end = preferred
+      }
+    }
+
+    const chunk = source.slice(cursor, end).trim()
+    if (chunk) {
+      chunks.push(chunk)
+    }
+    cursor = end
+  }
+
+  return chunks
+}
+
+function splitBodyByBudget(params: { body: string; budget: number; minChunkChars: number }): string[] {
+  const body = params.body.trim()
+  if (!body) return []
+  if (body.length <= params.budget) return [body]
+
+  const chunks: string[] = []
+  let cursor = 0
+  while (cursor < body.length) {
+    let end = Math.min(body.length, cursor + params.budget)
+    if (end < body.length) {
+      const breakAt = body.lastIndexOf('\n', end)
+      if (breakAt > cursor + Math.floor(params.minChunkChars / 2)) {
+        end = breakAt
+      }
+    }
+
+    const part = body.slice(cursor, end).trim()
+    if (part) {
+      chunks.push(part)
+    }
+    cursor = end
+  }
+
+  return chunks
+}
+
+function splitFileBody(
+  body: string,
+  options: { budget: number; minChunkChars: number; preserveCodeFenceIntegrity: boolean },
+): string[] {
+  const source = body.trim()
+  if (!source) return []
+
+  if (!options.preserveCodeFenceIntegrity) {
+    return splitBodyByBudget({ body: source, budget: options.budget, minChunkChars: options.minChunkChars })
+  }
+
+  const fenceHeaderMatch = source.match(/^```[^\n]*\n/)
+  const hasTrailingFence = source.endsWith('\n```')
+  if (!fenceHeaderMatch || !hasTrailingFence) {
+    return splitBodyByBudget({ body: source, budget: options.budget, minChunkChars: options.minChunkChars })
+  }
+
+  const openingFence = fenceHeaderMatch[0].trimEnd()
+  const innerStart = fenceHeaderMatch[0].length
+  const innerEnd = Math.max(innerStart, source.length - '\n```'.length)
+  const innerBody = source.slice(innerStart, innerEnd)
+  const wrapperChars = openingFence.length + '\n'.length + '\n```'.length
+  const innerBudget = Math.max(1_000, options.budget - wrapperChars)
+  const innerChunks = splitBodyByBudget({ body: innerBody, budget: innerBudget, minChunkChars: options.minChunkChars })
+
+  if (innerChunks.length === 0) {
+    return [source]
+  }
+
+  return innerChunks.map((chunk) => `${openingFence}\n${chunk}\n\`\`\``)
+}
+
+function mergeTinyTailParts(parts: string[], options: { minChunkChars: number; maxChars: number }): string[] {
+  if (parts.length <= 1) return parts
+  const merged = [...parts]
+  while (merged.length > 1) {
+    const last = merged[merged.length - 1] ?? ''
+    if (last.length >= options.minChunkChars) break
+    const previous = merged[merged.length - 2] ?? ''
+    const combinedLength = previous.length + SECTION_SEPARATOR_LENGTH + last.length
+    if (combinedLength > options.maxChars) break
+    merged.splice(merged.length - 2, 2, `${previous}\n\n${last}`.trim())
+  }
+  return merged
+}
+
+async function splitOversizedSection(
+  section: RepomixSection,
+  options: { maxChars: number; minChunkChars: number; preserveCodeFenceIntegrity: boolean },
+): Promise<RepomixSection[]> {
+  if (section.content.length <= options.maxChars) {
+    return [section]
+  }
+
+  if (section.kind !== 'file' || !section.filePath) {
+    const chunks = await splitTextByCharBudget(section.content, options.maxChars)
+    return chunks.map((content) => ({ kind: section.kind, content }))
+  }
+
+  const header = `## File: ${section.filePath}`
+  const body = section.content.startsWith(header) ? section.content.slice(header.length).trimStart() : section.content
+  const partPrefixTemplate = '\n(part 000/000)\n'
+  const bodyBudget = Math.max(1_000, options.maxChars - header.length - partPrefixTemplate.length)
+
+  const bodyParts = splitFileBody(body, {
+    budget: bodyBudget,
+    minChunkChars: options.minChunkChars,
+    preserveCodeFenceIntegrity: options.preserveCodeFenceIntegrity,
+  })
+  const normalizedParts = mergeTinyTailParts(bodyParts, { minChunkChars: options.minChunkChars, maxChars: bodyBudget })
+  const totalParts = Math.max(1, normalizedParts.length)
+
+  return normalizedParts.map((chunk, index) => ({
+    kind: 'file',
+    filePath: section.filePath,
+    content: `${header}\n(part ${index + 1}/${totalParts})\n${chunk}`.trim(),
+  }))
+}
+
+function parseRepomixSections(repomixOutput: string): RepomixSection[] {
+  const source = repomixOutput.trim()
+  if (!source) return []
+
+  const matches = Array.from(source.matchAll(new RegExp(FILE_SECTION_HEADER_SOURCE, 'gm')))
+  if (matches.length === 0) {
+    return [{ kind: 'preamble', content: source }]
+  }
+
+  const sections: RepomixSection[] = []
+  const firstMatch = matches.at(0)
+  const firstIndex = firstMatch ? firstMatch.index : 0
+  if (firstIndex > 0) {
+    const preamble = source.slice(0, firstIndex).trim()
+    if (preamble) {
+      sections.push({ kind: 'preamble', content: preamble })
+    }
+  }
+
+  for (const [index, match] of matches.entries()) {
+    const start = match.index
+    const nextStart = matches[index + 1]?.index ?? source.length
+    const content = source.slice(start, nextStart).trim()
+    if (!content) continue
+    const filePath = (match[1] ?? '').trim()
+    sections.push({ kind: 'file', content, filePath: filePath || undefined })
+  }
+
+  return sections
+}
+
+function mergeTinyChunks(
+  chunks: Omit<RepomixContextChunk, 'index' | 'totalChunks'>[],
+  options: { minChunkChars: number; maxChars: number },
+): Omit<RepomixContextChunk, 'index' | 'totalChunks'>[] {
+  if (chunks.length <= 1) return chunks
+  const merged: Omit<RepomixContextChunk, 'index' | 'totalChunks'>[] = []
+
+  for (const chunk of chunks) {
+    const previous = merged.at(-1)
+    if (
+      previous &&
+      chunk.charLength < options.minChunkChars &&
+      previous.charLength + SECTION_SEPARATOR_LENGTH + chunk.charLength <= options.maxChars
+    ) {
+      const combinedContent = `${previous.content}\n\n${chunk.content}`.trim()
+      merged[merged.length - 1] = {
+        ...previous,
+        content: combinedContent,
+        charLength: combinedContent.length,
+        tokenEstimate: estimateTokenCountFromChars(combinedContent),
+        sectionCount: previous.sectionCount + chunk.sectionCount,
+        fileCount: previous.fileCount + chunk.fileCount,
+        firstFilePath: previous.firstFilePath,
+        lastFilePath: chunk.lastFilePath ?? previous.lastFilePath,
+      }
+      continue
+    }
+    merged.push(chunk)
+  }
+
+  return merged
+}
+
+export async function chunkRepomixOutput(
+  repomixOutput: string,
+  options: RepomixChunkOptions = {},
+): Promise<RepomixContextChunk[]> {
+  const maxChars = normalizeMaxChars(options.maxChars)
+  const minChunkChars = normalizeMinChunkChars(options.minChunkChars, maxChars)
+  const preserveCodeFenceIntegrity = options.preserveCodeFenceIntegrity ?? true
+
+  const rawSections = parseRepomixSections(repomixOutput)
+  const splitSections = await Promise.all(
+    rawSections.map(
+      async (section) => await splitOversizedSection(section, { maxChars, minChunkChars, preserveCodeFenceIntegrity }),
+    ),
+  )
+  const sections = splitSections.flat()
+
+  if (sections.length === 0) return []
+
+  const chunks: Omit<RepomixContextChunk, 'index' | 'totalChunks'>[] = []
+  let currentParts: string[] = []
+  let currentCharLength = 0
+  let currentSectionCount = 0
+  let currentFileCount = 0
+  let currentFirstFilePath: string | null = null
+  let currentLastFilePath: string | null = null
+
+  const flushCurrent = () => {
+    if (currentParts.length === 0) return
+
+    const content = currentParts.join('\n\n').trim()
+    if (!content) return
+
+    chunks.push({
+      content,
+      charLength: content.length,
+      tokenEstimate: estimateTokenCountFromChars(content),
+      sectionCount: currentSectionCount,
+      fileCount: currentFileCount,
+      firstFilePath: currentFirstFilePath,
+      lastFilePath: currentLastFilePath,
+    })
+
+    currentParts = []
+    currentCharLength = 0
+    currentSectionCount = 0
+    currentFileCount = 0
+    currentFirstFilePath = null
+    currentLastFilePath = null
+  }
+
+  for (const section of sections) {
+    const sectionText = section.content.trim()
+    if (!sectionText) continue
+
+    const addedSeparator = currentParts.length > 0 ? SECTION_SEPARATOR_LENGTH : 0
+    const projected = currentCharLength + sectionText.length + addedSeparator
+    if (currentParts.length > 0 && projected > maxChars) {
+      flushCurrent()
+    }
+
+    currentParts.push(sectionText)
+    currentCharLength += sectionText.length + (currentParts.length > 1 ? SECTION_SEPARATOR_LENGTH : 0)
+    currentSectionCount += 1
+    if (section.kind === 'file') {
+      currentFileCount += 1
+      if (!currentFirstFilePath) {
+        currentFirstFilePath = section.filePath ?? null
+      }
+      currentLastFilePath = section.filePath ?? currentLastFilePath
+    }
+  }
+
+  flushCurrent()
+  const normalizedChunks = mergeTinyChunks(chunks, { minChunkChars, maxChars })
+  const totalChunks = normalizedChunks.length
+  return normalizedChunks.map((chunk, index) => ({ ...chunk, index: index + 1, totalChunks }))
+}