dsh-convert-core 0.1.0-alpha.1

package/src/contract.ts

@@ -0,0 +1,230 @@
+/**
+ * Wire-level contract for the conversion kernel.
+ *
+ * The kernel owns exactly one promise: given an input, produce a
+ * *knowledge-ready* Markdown bundle (`index.md` + `assets/` + `manifest.json`)
+ * plus an `ingest` payload that plugins/dsh-data-domain-knowledge can consume
+ * without the two ever importing each other.
+ *
+ * Boundary rule (issue 0013): the kernel never decides `source_type`. It only
+ * proposes candidates; the ingestion side decides and records
+ * `source_type_source`.
+ */
+
+/**
+ * `source_format` values from the 0013 contract. The full vocabulary is kept so
+ * the front matter stays wire-compatible, but this version only ever emits
+ * `markdown`, `text`, `pdf` and `office`.
+ */
+export const SOURCE_FORMATS = [
+  'markdown',
+  'text',
+  'html',
+  'pdf',
+  'office',
+  'web_page',
+  'video',
+  'audio',
+  'image',
+] as const
+
+export type SourceFormat = (typeof SOURCE_FORMATS)[number]
+
+/**
+ * This version is documents-only (issue 0015 scope cut): web pages and video
+ * moved to issues/0017, so only `file` and `text` can be requested.
+ */
+export const CONVERT_INPUT_KINDS = ['file', 'text'] as const
+
+export type ConvertInputKind = (typeof CONVERT_INPUT_KINDS)[number]
+
+export const TASK_STATES = ['queued', 'running', 'succeeded', 'failed', 'cancelled'] as const
+
+export type TaskState = (typeof TASK_STATES)[number]
+
+/** Lineage mirrors the knowledge plugin so both ends speak one vocabulary. */
+export interface Lineage {
+  sessionId?: string
+  turnId?: string
+  intentId?: string
+  taskId?: string
+  subtaskId?: string
+  parentSubtaskId?: string
+  runId?: string
+  traceId?: string
+  domain?: string
+}
+
+export type SourceTypeSource = 'agent' | 'user' | 'default'
+
+/**
+ * Front matter written into `index.md`.
+ *
+ * Required by the knowledge-ready contract: `title`, `created_at`,
+ * `updated_at`. `source_id` and `source_type` are optional here because the
+ * kernel must not invent them — ingestion fills them in and re-writes the file.
+ */
+export interface ConversionFrontMatter {
+  source_id?: string
+  source_type?: string
+  title: string
+  created_at: string
+  updated_at: string
+  external_id?: string
+  source_format?: SourceFormat
+  content_hash?: string
+  parser?: string
+  source_url?: string
+  source_path?: string
+  source_type_source?: SourceTypeSource
+  source_type_candidates?: string[]
+  word_count?: number
+  asset_count?: number
+  lineage?: Lineage
+}
+
+export interface ConvertAsset {
+  /** Path relative to the delivery directory, e.g. `assets/img-1a2b3c4d.png`. */
+  rel: string
+  /** Absolute path on disk inside the delivery directory. */
+  path: string
+  bytes: number
+  contentType?: string
+  sha1: string
+  /** Original remote URL when the asset was downloaded. */
+  sourceUrl?: string
+  /** `inline` when it came from a data: URI, `remote` when downloaded, `local` when copied. */
+  origin: 'remote' | 'inline' | 'local'
+}
+
+export interface ConvertWarning {
+  code: string
+  message: string
+}
+
+/** Everything an ingestion side needs to file this bundle, without guessing. */
+export interface IngestPayload {
+  /** Markdown body with a leading front matter block, ready for `add_source`. */
+  markdown: string
+  /** Same body without front matter. */
+  content: string
+  title: string
+  externalId: string
+  contentHash: string
+  sourceFormat: SourceFormat
+  parser: string
+  sourceUrl?: string
+  sourcePath?: string
+  sourceTypeCandidates: string[]
+  /** True when nobody supplied a `source_type` yet. */
+  sourceTypeRequired: boolean
+  assets: ConvertAsset[]
+  /**
+   * Image references are relative to the delivery `index.md`. A wiki card lives
+   * in `02_wiki/<type>/`, so ingestion must re-base them before a card can feed
+   * the multimodal embedder.
+   */
+  assetRefBase: 'delivery'
+}
+
+export interface ConvertManifest {
+  version: 1
+  taskId: string
+  inputKind: ConvertInputKind
+  sourceFormat: SourceFormat
+  parser: string
+  title: string
+  createdAt: string
+  updatedAt: string
+  deliveryDir: string
+  contentHash: string
+  externalId: string
+  wordCount: number
+  assets: ConvertAsset[]
+  warnings: ConvertWarning[]
+  /** Version of the external binary / service that did the work, when any. */
+  dependencyVersions: Record<string, string>
+  lineage?: Lineage
+  ingest: IngestPayload
+}
+
+export interface ConvertTask {
+  id: string
+  state: TaskState
+  inputKind: ConvertInputKind
+  /** Human label shown in the console. */
+  label: string
+  sourceFormat: SourceFormat
+  createdAt: string
+  updatedAt: string
+  startedAt?: string
+  finishedAt?: string
+  /** 0..1, or null when the adapter cannot report a ratio. */
+  progress: number | null
+  stage: string
+  message?: string
+  error?: { code: string; message: string; suggestion?: string }
+  warnings: ConvertWarning[]
+  deliveryDir?: string
+  manifest?: ConvertManifest
+  attempts: number
+  /** Absolute input path inside the kernel work dir, when the input was a file. */
+  inputPath?: string
+  request: ConvertRequest
+}
+
+export interface ConvertRequest {
+  kind: ConvertInputKind
+  /** Absolute path on the host for kind=file|video; the kernel reads and stages it. */
+  localPath?: string
+  /** File name inside the kernel work dir (kind=file|video). */
+  filename?: string
+  /** Inline text (kind=text). */
+  text?: string
+  title?: string
+  /** Caller-supplied business classification; the kernel never invents one. */
+  sourceType?: string
+  sourceTypeCandidates?: string[]
+  sourceTypeSource?: SourceTypeSource
+  /** Stable identity for idempotency, e.g. a canonical URL or a file hash. */
+  externalId?: string
+  lineage?: Lineage
+  /** Adapter override, mostly for tests and for the high-quality tier. */
+  adapter?: string
+}
+
+export interface ConvertError {
+  code: string
+  message: string
+  suggestion?: string
+}
+
+export interface ConvertResult {
+  task: ConvertTask
+}
+
+export function isSourceFormat(value: unknown): value is SourceFormat {
+  return typeof value === 'string' && (SOURCE_FORMATS as readonly string[]).includes(value)
+}
+
+/** Default business-classification hints per carrier format. */
+export function defaultSourceTypeCandidates(format: SourceFormat): string[] {
+  switch (format) {
+    case 'pdf':
+      return ['article', 'book', 'prd', 'document']
+    case 'office':
+      return ['prd', 'document']
+    case 'markdown':
+    case 'text':
+      return ['document', 'article']
+    // iddoc 档(issue 0017):音视频与图片的业务归类与文档不同,别让它们默认落到 "document"。
+    case 'video':
+      return ['video', 'course', 'interview', 'document']
+    case 'audio':
+      return ['audio', 'interview', 'meeting', 'document']
+    case 'image':
+      return ['image', 'screenshot', 'diagram', 'document']
+    default:
+      return ['document']
+  }
+}

package/src/delivery.ts

@@ -0,0 +1,124 @@
+/**
+ * Delivery bundle writer.
+ *
+ * Layout is fixed by the 0013 knowledge-ready contract:
+ *
+ *   convert-out/<taskId>/
+ *     index.md          front matter + body, images referenced as assets/…
+ *     assets/img-*.ext  content-addressed local images
+ *     manifest.json     provenance, parser/version, warnings, ingest payload
+ *
+ * The kernel never writes into the knowledge store; it only produces this
+ * directory plus the `ingest` payload that the DSH layer forwards.
+ */
+
+import { mkdir } from 'node:fs/promises'
+import { join } from 'node:path'
+import type {
+  ConvertAsset,
+  ConvertManifest,
+  ConvertRequest,
+  ConvertWarning,
+  IngestPayload,
+  SourceFormat,
+  SourceTypeSource,
+} from './contract.js'
+import { defaultSourceTypeCandidates } from './contract.js'
+import { sha256, toMarkdownDocument, wordCount, writeTextAtomic } from './markdown.js'
+
+export interface DeliveryInput {
+  taskId: string
+  title: string
+  request: ConvertRequest
+  /** Markdown body with image references relative to `index.md`. */
+  markdown: string
+  sourceFormat: SourceFormat
+  parser: string
+  sourceUrl?: string
+  sourcePath?: string
+  assets: ConvertAsset[]
+  warnings: ConvertWarning[]
+  dependencyVersions: Record<string, string>
+  externalId: string
+  createdAt: string
+  updatedAt: string
+}
+
+export function buildFrontMatter(input: DeliveryInput, contentHash: string): Record<string, unknown> {
+  const candidates = input.request.sourceTypeCandidates?.length
+    ? input.request.sourceTypeCandidates
+    : defaultSourceTypeCandidates(input.sourceFormat)
+  const frontMatter: Record<string, unknown> = {
+    title: input.title,
+    created_at: input.createdAt,
+    updated_at: input.updatedAt,
+    external_id: input.externalId,
+    source_format: input.sourceFormat,
+    content_hash: contentHash,
+    parser: input.parser,
+    source_type_candidates: candidates,
+    word_count: wordCount(input.markdown),
+    asset_count: input.assets.length,
+  }
+  if (input.request.sourceType) frontMatter.source_type = input.request.sourceType
+  if (input.sourceUrl) frontMatter.source_url = input.sourceUrl
+  if (input.sourcePath) frontMatter.source_path = input.sourcePath
+  if (input.request.sourceTypeSource) frontMatter.source_type_source = input.request.sourceTypeSource
+  if (input.request.lineage && Object.keys(input.request.lineage).length > 0) frontMatter.lineage = input.request.lineage
+  return frontMatter
+}
+
+export function buildIngestPayload(input: DeliveryInput, contentHash: string): IngestPayload {
+  const candidates = input.request.sourceTypeCandidates?.length
+    ? input.request.sourceTypeCandidates
+    : defaultSourceTypeCandidates(input.sourceFormat)
+  return {
+    markdown: toMarkdownDocument(buildFrontMatter(input, contentHash), input.markdown),
+    content: input.markdown,
+    title: input.title,
+    externalId: input.externalId,
+    contentHash,
+    sourceFormat: input.sourceFormat,
+    parser: input.parser,
+    ...(input.sourceUrl ? { sourceUrl: input.sourceUrl } : {}),
+    ...(input.sourcePath ? { sourcePath: input.sourcePath } : {}),
+    sourceTypeCandidates: candidates,
+    sourceTypeRequired: !input.request.sourceType,
+    assets: input.assets,
+    assetRefBase: 'delivery',
+  }
+}
+
+export async function writeDelivery(deliveryDir: string, input: DeliveryInput): Promise<ConvertManifest> {
+  await mkdir(deliveryDir, { recursive: true })
+  const contentHash = sha256(input.markdown)
+  const frontMatter = buildFrontMatter(input, contentHash)
+  const document = toMarkdownDocument(frontMatter, input.markdown)
+  await writeTextAtomic(join(deliveryDir, 'index.md'), document)
+
+  const manifest: ConvertManifest = {
+    version: 1,
+    taskId: input.taskId,
+    inputKind: input.request.kind,
+    sourceFormat: input.sourceFormat,
+    parser: input.parser,
+    title: input.title,
+    createdAt: input.createdAt,
+    updatedAt: input.updatedAt,
+    deliveryDir,
+    contentHash,
+    externalId: input.externalId,
+    wordCount: wordCount(input.markdown),
+    assets: input.assets,
+    warnings: input.warnings,
+    dependencyVersions: input.dependencyVersions,
+    ...(input.request.lineage ? { lineage: input.request.lineage } : {}),
+    ingest: buildIngestPayload(input, contentHash),
+  }
+  await writeTextAtomic(join(deliveryDir, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`)
+  return manifest
+}
+
+export function sourceTypeSourceOf(value: string | undefined): SourceTypeSource | undefined {
+  return value === 'agent' || value === 'user' || value === 'default' ? value : undefined
+}