@alta-foundation/plaud-extractor 1.0.0

package/src/sync/sync-engine.ts

@@ -0,0 +1,183 @@
+import { AuthError } from '../errors.js'
+import { getLogger } from '../logger.js'
+import { IncrementalTracker } from './incremental.js'
+import { processQueue, retryWithBackoff } from './download-queue.js'
+import { RecordingStore } from '../storage/recording-store.js'
+import { DatasetWriter } from '../storage/dataset-writer.js'
+import type { SyncOptions, SyncResult, BackfillOptions } from './types.js'
+import type { PlaudClient, PlaudRecording } from '../client/types.js'
+import type { HttpClient } from '../client/http.js'
+
+export class SyncEngine {
+  async run(
+    client: PlaudClient,
+    opts: SyncOptions,
+    mode: 'sync' | 'backfill' = 'sync',
+  ): Promise<SyncResult> {
+    const log = getLogger()
+    const startedAt = Date.now()
+
+    // 1. Verify auth
+    const authed = await client.isAuthenticated()
+    if (!authed) {
+      throw new AuthError("Not authenticated — run 'alta-plaud auth' first")
+    }
+
+    // 2. Load sync state
+    const tracker = new IncrementalTracker()
+    await tracker.load(opts.outDir)
+
+    // 3. Determine effective --since
+    const since = opts.since ?? (mode === 'sync' ? tracker.getSince() : undefined)
+
+    log.info({ mode, since: since?.toISOString(), outDir: opts.outDir }, 'Starting sync')
+
+    // 4. Collect recordings to process
+    const toProcess: PlaudRecording[] = []
+    const skipped: PlaudRecording[] = []
+    let listCount = 0
+
+    for await (const recording of client.listRecordings({ since, limit: opts.limit })) {
+      listCount++
+      if (mode === 'sync' && !tracker.needsDownload(recording)) {
+        skipped.push(recording)
+      } else {
+        toProcess.push(recording)
+      }
+
+      if (opts.limit && toProcess.length >= opts.limit) break
+    }
+
+    log.info(
+      { total: listCount, toProcess: toProcess.length, skipped: skipped.length },
+      'Recordings collected',
+    )
+
+    // 5. Dry run — just print plan
+    if (opts.dryRun) {
+      for (const rec of toProcess) {
+        log.info(
+          { id: rec.id, title: rec.title, recordedAt: rec.recordedAt },
+          '[dry-run] Would download',
+        )
+      }
+      return {
+        mode,
+        attempted: 0,
+        succeeded: 0,
+        failed: 0,
+        skipped: skipped.length,
+        durationMs: Date.now() - startedAt,
+        errors: [],
+      }
+    }
+
+    // 6. Initialize storage
+    const store = new RecordingStore(opts.outDir)
+    const dataset = opts.includeDataset ? new DatasetWriter(opts.outDir) : null
+    if (dataset) await dataset.open()
+
+    // 7. Run download queue
+    const errors: Array<{ recordingId: string; error: Error }> = []
+    const httpClient = getHttpClient(client)
+
+    const { succeeded, failed } = await processQueue(
+      toProcess,
+      async (recording) => {
+        await retryWithBackoff(
+          () => downloadRecording(recording, client, store, dataset, tracker, httpClient, opts),
+          { label: `recording:${recording.id}` },
+        )
+      },
+      opts.concurrency,
+    )
+
+    for (const { item, error } of failed) {
+      errors.push({ recordingId: item.id, error })
+      log.error({ recordingId: item.id, err: error }, 'Failed to download recording')
+    }
+
+    // 8. Mark successful sync only if zero failures
+    if (errors.length === 0) {
+      tracker.markSuccessfulSync()
+    }
+
+    // 9. Persist state
+    await tracker.persist(opts.outDir)
+
+    if (dataset) await dataset.close()
+
+    const result: SyncResult = {
+      mode,
+      attempted: toProcess.length,
+      succeeded: succeeded.length,
+      failed: failed.length,
+      skipped: skipped.length,
+      durationMs: Date.now() - startedAt,
+      errors,
+      datasetPath: dataset?.path,
+    }
+
+    log.info(result, 'Sync complete')
+    return result
+  }
+}
+
+async function downloadRecording(
+  recording: PlaudRecording,
+  client: PlaudClient,
+  store: RecordingStore,
+  dataset: DatasetWriter | null,
+  tracker: IncrementalTracker,
+  httpClient: HttpClient | null,
+  opts: SyncOptions,
+): Promise<void> {
+  const log = getLogger()
+  log.info({ recordingId: recording.id, title: recording.title }, 'Downloading recording')
+
+  // a. Write metadata
+  await store.writeMetadata(recording)
+
+  // b. Download transcript
+  let hasTranscript = false
+  if (recording.hasTranscript) {
+    try {
+      const transcript = await client.getTranscript(recording.id)
+      await store.writeTranscript(recording, transcript, opts.formats)
+      if (dataset) {
+        await dataset.append(opts.outDir, recording, transcript)
+      }
+      hasTranscript = true
+    } catch (err) {
+      log.warn({ recordingId: recording.id, err }, 'Failed to get transcript')
+    }
+  }
+
+  // c. Download audio
+  let hasAudio = false
+  if (httpClient) {
+    const audioUrl = await client.getAudioDownloadUrl(recording.id)
+    if (audioUrl) {
+      hasAudio = await store.writeAudioFromUrl(recording, audioUrl, httpClient)
+    }
+  }
+
+  // d. Write checksums
+  await store.writeChecksums(recording)
+
+  // e. Mark complete in state
+  tracker.markComplete(recording.id, recording.recordedAt, {
+    hasAudio,
+    hasTranscript,
+    contentHash: tracker.computeContentHash(recording),
+  })
+}
+
+/** Extract HttpClient from PlaudApiClient for audio downloads */
+function getHttpClient(client: PlaudClient): HttpClient | null {
+  // PlaudApiClient exposes getHttpClient()
+  if ('getHttpClient' in client && typeof (client as { getHttpClient?: () => HttpClient }).getHttpClient === 'function') {
+    return (client as { getHttpClient: () => HttpClient }).getHttpClient()
+  }
+  return null
+}

package/src/sync/types.ts

@@ -0,0 +1,64 @@
+import { z } from 'zod'
+import type { TranscriptFormat } from '../storage/recording-store.js'
+
+export const RecordingStateSchema = z.object({
+  recordedAt: z.string().datetime(),
+  contentHash: z.string().optional(),
+  downloadedAt: z.string().datetime().optional(),
+  hasAudio: z.boolean().default(false),
+  hasTranscript: z.boolean().default(false),
+  verified: z.boolean().default(false),
+  verifiedAt: z.string().datetime().optional(),
+})
+
+export type RecordingState = z.infer<typeof RecordingStateSchema>
+
+export const SyncStateSchema = z.object({
+  schemaVersion: z.literal(1),
+  lastSuccessfulSyncAt: z.string().datetime().optional(),
+  lastAttemptAt: z.string().datetime().optional(),
+  recordings: z.record(z.string(), RecordingStateSchema),
+})
+
+export type SyncState = z.infer<typeof SyncStateSchema>
+
+export interface SyncOptions {
+  /** Output directory root */
+  outDir: string
+  /** Only sync recordings after this date */
+  since?: Date
+  /** Max number of recordings to process */
+  limit?: number
+  /** Parallel downloads (default: 3) */
+  concurrency: number
+  /** Transcript formats to write */
+  formats: TranscriptFormat[]
+  /** Append to JSONL dataset */
+  includeDataset: boolean
+  /** Print plan without downloading */
+  dryRun: boolean
+}
+
+export interface BackfillOptions extends Omit<SyncOptions, 'since'> {
+  /** Backfill from a specific date; defaults to all-time */
+  since?: Date
+}
+
+export interface SyncResult {
+  mode: 'sync' | 'backfill'
+  attempted: number
+  succeeded: number
+  failed: number
+  skipped: number
+  durationMs: number
+  errors: Array<{ recordingId: string; error: Error }>
+  datasetPath?: string
+}
+
+export interface VerifyResult {
+  scanned: number
+  ok: number
+  failed: number
+  repaired: number
+  issues: Array<{ recordingId: string; file: string; issue: string }>
+}

package/src/transcript/formatter.ts

@@ -0,0 +1,91 @@
+import type { PlaudTranscript, PlaudRecording } from '../client/types.js'
+
+export function toPlainText(transcript: PlaudTranscript): string {
+  return transcript.segments
+    .map(seg => {
+      const speaker = seg.speaker ? `${seg.speaker}: ` : ''
+      return `${speaker}${seg.text}`
+    })
+    .join('\n\n')
+}
+
+export function toMarkdown(transcript: PlaudTranscript, recording: PlaudRecording): string {
+  const lines: string[] = []
+
+  // YAML frontmatter
+  lines.push('---')
+  lines.push('source: plaud')
+  lines.push(`id: "${recording.id}"`)
+  lines.push(`recorded_at: "${recording.recordedAt}"`)
+  if (recording.title) lines.push(`title: "${recording.title.replace(/"/g, '\\"')}"`)
+  if (recording.language) lines.push(`language: "${recording.language}"`)
+  lines.push(`duration_seconds: ${recording.duration}`)
+  if (recording.tags?.length) lines.push(`tags: [${recording.tags.map(t => `"${t}"`).join(', ')}]`)
+  lines.push('---')
+  lines.push('')
+
+  // Title
+  lines.push(`# ${recording.title ?? 'Untitled Recording'}`)
+  lines.push('')
+
+  // Metadata block
+  lines.push(`**Recorded:** ${formatDate(recording.recordedAt)}`)
+  lines.push(`**Duration:** ${formatDuration(recording.duration)}`)
+  if (recording.language) lines.push(`**Language:** ${recording.language}`)
+  lines.push('')
+  lines.push('## Transcript')
+  lines.push('')
+
+  // Segments
+  const hasTimestamps = transcript.segments.some(s => s.startMs > 0 || s.endMs > 0)
+
+  for (const seg of transcript.segments) {
+    if (hasTimestamps) {
+      const ts = `\`[${msToTimestamp(seg.startMs)}]\``
+      const speaker = seg.speaker ? ` **${seg.speaker}**` : ''
+      lines.push(`${ts}${speaker}`)
+    } else if (seg.speaker) {
+      lines.push(`**${seg.speaker}**`)
+    }
+    lines.push(seg.text)
+    lines.push('')
+  }
+
+  return lines.join('\n')
+}
+
+function msToTimestamp(ms: number): string {
+  const totalSeconds = Math.floor(ms / 1000)
+  const h = Math.floor(totalSeconds / 3600)
+  const m = Math.floor((totalSeconds % 3600) / 60)
+  const s = totalSeconds % 60
+  if (h > 0) {
+    return `${h}:${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}`
+  }
+  return `${m}:${String(s).padStart(2, '0')}`
+}
+
+function formatDuration(seconds: number): string {
+  const h = Math.floor(seconds / 3600)
+  const m = Math.floor((seconds % 3600) / 60)
+  const s = Math.floor(seconds % 60)
+  if (h > 0) return `${h}h ${m}m ${s}s`
+  if (m > 0) return `${m}m ${s}s`
+  return `${s}s`
+}
+
+function formatDate(iso: string): string {
+  try {
+    return new Date(iso).toLocaleString('en-US', {
+      weekday: 'long',
+      year: 'numeric',
+      month: 'long',
+      day: 'numeric',
+      hour: '2-digit',
+      minute: '2-digit',
+      timeZoneName: 'short',
+    })
+  } catch {
+    return iso
+  }
+}

package/tsconfig.build.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true
+  }
+}

package/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "lib": ["ES2022"],
+    "outDir": "dist",
+    "rootDir": "src",
+    "strict": true,
+    "noUncheckedIndexedAccess": true,
+    "forceConsistentCasingInFileNames": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "skipLibCheck": true
+  },
+  "include": ["src"],
+  "exclude": ["node_modules", "dist"]
+}