@alta-foundation/plaud-extractor 1.0.0

package/src/storage/checksums.ts

@@ -0,0 +1,76 @@
+import crypto from 'node:crypto'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import { writeFileAtomic } from './atomic.js'
+import { ChecksumMismatchError } from '../errors.js'
+
+export interface FileChecksum {
+  sha256: string
+  sizeBytes: number
+}
+
+export interface ChecksumManifest {
+  schemaVersion: 1
+  recordingId: string
+  computedAt: string
+  files: Record<string, FileChecksum>
+}
+
+export async function sha256File(filePath: string): Promise<string> {
+  const data = await fs.readFile(filePath)
+  return crypto.createHash('sha256').update(data).digest('hex')
+}
+
+export async function writeChecksumManifest(
+  dir: string,
+  recordingId: string,
+): Promise<ChecksumManifest> {
+  const files = await fs.readdir(dir)
+  const manifest: ChecksumManifest = {
+    schemaVersion: 1,
+    recordingId,
+    computedAt: new Date().toISOString(),
+    files: {},
+  }
+
+  for (const file of files.filter(f => f !== 'checksums.json')) {
+    const fPath = path.join(dir, file)
+    const stat = await fs.stat(fPath)
+    if (!stat.isFile()) continue
+    manifest.files[file] = {
+      sha256: await sha256File(fPath),
+      sizeBytes: stat.size,
+    }
+  }
+
+  await writeFileAtomic(path.join(dir, 'checksums.json'), JSON.stringify(manifest, null, 2))
+  return manifest
+}
+
+export async function verifyChecksums(dir: string): Promise<ChecksumMismatchError[]> {
+  const manifestPath = path.join(dir, 'checksums.json')
+  let manifest: ChecksumManifest
+
+  try {
+    const raw = await fs.readFile(manifestPath, 'utf8')
+    manifest = JSON.parse(raw) as ChecksumManifest
+  } catch {
+    return [] // No manifest yet — skip verification
+  }
+
+  const mismatches: ChecksumMismatchError[] = []
+
+  for (const [file, expected] of Object.entries(manifest.files)) {
+    const fPath = path.join(dir, file)
+    try {
+      const actual = await sha256File(fPath)
+      if (actual !== expected.sha256) {
+        mismatches.push(new ChecksumMismatchError(fPath, expected.sha256, actual))
+      }
+    } catch {
+      mismatches.push(new ChecksumMismatchError(fPath, expected.sha256, 'MISSING'))
+    }
+  }
+
+  return mismatches
+}

package/src/storage/dataset-writer.ts

@@ -0,0 +1,74 @@
+import fs from 'node:fs/promises'
+import { createWriteStream, type WriteStream } from 'node:fs'
+import path from 'node:path'
+import { datasetPath, recordingDir } from './paths.js'
+import type { PlaudRecording, PlaudTranscript } from '../client/types.js'
+import { StorageError } from '../errors.js'
+
+export interface DatasetEntry {
+  id: string
+  title: string | null
+  recorded_at: string
+  duration_seconds: number
+  language: string | null
+  text: string
+  path: string
+  segment_count: number
+}
+
+export class DatasetWriter {
+  private readonly filePath: string
+  private stream: WriteStream | null = null
+
+  constructor(outDir: string) {
+    this.filePath = datasetPath(outDir)
+  }
+
+  async open(): Promise<void> {
+    await fs.mkdir(path.dirname(this.filePath), { recursive: true })
+    this.stream = createWriteStream(this.filePath, { flags: 'a', encoding: 'utf8' })
+    await new Promise<void>((resolve, reject) => {
+      this.stream!.once('open', () => resolve())
+      this.stream!.once('error', reject)
+    })
+  }
+
+  async append(outDir: string, recording: PlaudRecording, transcript: PlaudTranscript): Promise<void> {
+    if (!this.stream) throw new StorageError('DatasetWriter not opened', this.filePath)
+
+    const relPath = path.relative(
+      outDir,
+      path.join(recordingDir(outDir, recording.recordedAt, recording.id), 'transcript.txt'),
+    )
+
+    const entry: DatasetEntry = {
+      id: `plaud:${recording.id}`,
+      title: recording.title ?? null,
+      recorded_at: recording.recordedAt,
+      duration_seconds: recording.duration,
+      language: recording.language ?? null,
+      text: transcript.fullText,
+      path: relPath,
+      segment_count: transcript.segments.length,
+    }
+
+    const line = JSON.stringify(entry) + '\n'
+
+    await new Promise<void>((resolve, reject) => {
+      this.stream!.write(line, err => (err ? reject(err) : resolve()))
+    })
+  }
+
+  async close(): Promise<void> {
+    if (!this.stream) return
+    await new Promise<void>((resolve, reject) => {
+      this.stream!.end(() => resolve())
+      this.stream!.once('error', reject)
+    })
+    this.stream = null
+  }
+
+  get path(): string {
+    return this.filePath
+  }
+}

package/src/storage/paths.ts

@@ -0,0 +1,44 @@
+import path from 'node:path'
+import os from 'node:os'
+
+export function defaultOutDir(): string {
+  const env = process.env['ALTA_DATA_DIR']
+  if (env) return path.resolve(env)
+  return path.join(os.homedir(), 'alta', 'data', 'plaud')
+}
+
+/** plaud/recordings/2026/02/2026-02-24T083012Z__plaud_<id>/ */
+export function recordingDir(outDir: string, recordedAt: string, recordingId: string): string {
+  const dt = new Date(recordedAt)
+  const year = String(dt.getUTCFullYear())
+  const month = String(dt.getUTCMonth() + 1).padStart(2, '0')
+  const timestamp = formatISOCompact(dt)
+  const dirName = `${timestamp}__plaud_${recordingId}`
+  return path.join(outDir, 'recordings', year, month, dirName)
+}
+
+/** Format: 2026-02-24T083012Z */
+function formatISOCompact(dt: Date): string {
+  const iso = dt.toISOString() // 2026-02-24T08:30:12.000Z
+  return iso.replace(/[-:]/g, '').replace(/\.\d{3}/, '').replace('T', 'T').slice(0, 16) + 'Z'
+}
+
+export function stateDir(outDir: string): string {
+  return path.join(outDir, '_state')
+}
+
+export function syncStatePath(outDir: string): string {
+  return path.join(stateDir(outDir), 'sync_state.json')
+}
+
+export function runLogsPath(outDir: string): string {
+  return path.join(stateDir(outDir), 'run_logs.ndjson')
+}
+
+export function datasetPath(outDir: string): string {
+  return path.join(outDir, 'datasets', 'plaud_transcripts.jsonl')
+}
+
+export function authTokenPath(): string {
+  return path.join(os.homedir(), '.alta', 'plaud-auth.json')
+}

package/src/storage/recording-store.ts

@@ -0,0 +1,182 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import { writeFileAtomic, writeStreamAtomic } from './atomic.js'
+import { writeChecksumManifest, verifyChecksums } from './checksums.js'
+import { toPlainText, toMarkdown } from '../transcript/formatter.js'
+import { recordingDir } from './paths.js'
+import type { PlaudRecording, PlaudTranscript } from '../client/types.js'
+import type { ChecksumMismatchError } from '../errors.js'
+import type { HttpClient } from '../client/http.js'
+import { getLogger } from '../logger.js'
+
+export type TranscriptFormat = 'json' | 'txt' | 'md'
+
+export interface RecordingWriteResult {
+  dir: string
+  hasAudio: boolean
+  hasTranscript: boolean
+}
+
+export class RecordingStore {
+  constructor(private readonly outDir: string) {}
+
+  recordingDir(recording: PlaudRecording): string {
+    return recordingDir(this.outDir, recording.recordedAt, recording.id)
+  }
+
+  async writeMetadata(recording: PlaudRecording): Promise<string> {
+    const dir = this.recordingDir(recording)
+    const meta = buildMetaJson(recording)
+    await writeFileAtomic(path.join(dir, 'meta.json'), JSON.stringify(meta, null, 2))
+    getLogger().debug({ recordingId: recording.id, dir }, 'Wrote meta.json')
+    return dir
+  }
+
+  async writeTranscript(
+    recording: PlaudRecording,
+    transcript: PlaudTranscript,
+    formats: TranscriptFormat[] = ['json', 'txt', 'md'],
+  ): Promise<void> {
+    const dir = this.recordingDir(recording)
+
+    if (formats.includes('json')) {
+      const json = {
+        recordingId: transcript.recordingId,
+        language: transcript.language,
+        duration: transcript.duration,
+        segments: transcript.segments,
+      }
+      await writeFileAtomic(path.join(dir, 'transcript.json'), JSON.stringify(json, null, 2))
+    }
+
+    if (formats.includes('txt')) {
+      const txt = toPlainText(transcript)
+      await writeFileAtomic(path.join(dir, 'transcript.txt'), txt)
+    }
+
+    if (formats.includes('md')) {
+      const md = toMarkdown(transcript, recording)
+      await writeFileAtomic(path.join(dir, 'transcript.md'), md)
+    }
+
+    getLogger().debug({ recordingId: recording.id, formats }, 'Wrote transcript files')
+  }
+
+  async writeAudio(recording: PlaudRecording, http: HttpClient): Promise<boolean> {
+    const url = await this.getAudioUrl(recording, http)
+    if (!url) return false
+
+    const dir = this.recordingDir(recording)
+    const ext = guessAudioExtension(recording.mimeType)
+    const destPath = path.join(dir, `audio.${ext}`)
+
+    try {
+      const stream = await http.getStream(url)
+      await writeStreamAtomic(destPath, stream)
+      getLogger().debug({ recordingId: recording.id, path: destPath }, 'Wrote audio file')
+      return true
+    } catch (err) {
+      getLogger().warn({ recordingId: recording.id, err }, 'Failed to download audio')
+      return false
+    }
+  }
+
+  private async getAudioUrl(recording: PlaudRecording, http: HttpClient): Promise<string | null> {
+    // This is a stub — the PlaudApiClient resolves the actual URL
+    // RecordingStore receives the already-resolved URL via writeAudioFromUrl
+    return null
+  }
+
+  async writeAudioFromUrl(recording: PlaudRecording, url: string, http: HttpClient): Promise<boolean> {
+    const dir = this.recordingDir(recording)
+    const ext = guessAudioExtension(recording.mimeType)
+    const destPath = path.join(dir, `audio.${ext}`)
+
+    try {
+      // S3 presigned URLs must be fetched without Plaud auth headers
+      const stream = await http.downloadExternalUrl(url)
+      await writeStreamAtomic(destPath, stream)
+      getLogger().debug({ recordingId: recording.id, path: destPath }, 'Wrote audio file')
+      return true
+    } catch (err) {
+      getLogger().warn({ recordingId: recording.id, err }, 'Failed to download audio')
+      return false
+    }
+  }
+
+  async writeChecksums(recording: PlaudRecording): Promise<void> {
+    const dir = this.recordingDir(recording)
+    await writeChecksumManifest(dir, recording.id)
+    getLogger().debug({ recordingId: recording.id }, 'Wrote checksums.json')
+  }
+
+  async verify(recording: PlaudRecording): Promise<ChecksumMismatchError[]> {
+    const dir = this.recordingDir(recording)
+    return verifyChecksums(dir)
+  }
+
+  async exists(recording: PlaudRecording): Promise<boolean> {
+    const dir = this.recordingDir(recording)
+    try {
+      await fs.access(dir)
+      return true
+    } catch {
+      return false
+    }
+  }
+
+  async hasMissingFiles(recording: PlaudRecording): Promise<boolean> {
+    const dir = this.recordingDir(recording)
+    const required = ['meta.json']
+    for (const file of required) {
+      try {
+        await fs.access(path.join(dir, file))
+      } catch {
+        return true
+      }
+    }
+    return false
+  }
+}
+
+function buildMetaJson(recording: PlaudRecording): object {
+  return {
+    source: 'plaud',
+    source_recording_id: recording.id,
+    recorded_at: recording.recordedAt,
+    imported_at: new Date().toISOString(),
+    title: recording.title,
+    duration_seconds: recording.duration,
+    language: recording.language,
+    audio: recording.fileSize
+      ? {
+          filename: `audio.${guessAudioExtension(recording.mimeType)}`,
+          mime: recording.mimeType,
+          bytes: recording.fileSize,
+        }
+      : null,
+    transcript: {
+      has_timestamps: true, // will be updated after writing
+      format: 'segments',
+      filename_json: 'transcript.json',
+      filename_txt: 'transcript.txt',
+      filename_md: 'transcript.md',
+    },
+    integrity: {
+      dedupe_key: `plaud:${recording.id}`,
+    },
+    tags: recording.tags,
+    folder_id: recording.folderId,
+    device_id: recording.deviceId,
+    summary: recording.summary,
+  }
+}
+
+function guessAudioExtension(mimeType: string): string {
+  if (mimeType.includes('m4a') || mimeType.includes('mp4')) return 'm4a'
+  if (mimeType.includes('wav')) return 'wav'
+  if (mimeType.includes('mp3')) return 'mp3'
+  if (mimeType.includes('ogg')) return 'ogg'
+  if (mimeType.includes('webm')) return 'webm'
+  return 'm4a' // default
+}

package/src/sync/download-queue.ts

@@ -0,0 +1,102 @@
+import { ApiError } from '../errors.js'
+import { getLogger } from '../logger.js'
+
+export interface QueueResult<T> {
+  succeeded: T[]
+  failed: Array<{ item: T; error: Error }>
+}
+
+/**
+ * Process items with bounded concurrency.
+ * Uses a simple semaphore — no extra dependencies.
+ */
+export async function processQueue<T>(
+  items: T[],
+  processor: (item: T) => Promise<void>,
+  concurrency: number,
+): Promise<QueueResult<T>> {
+  const succeeded: T[] = []
+  const failed: Array<{ item: T; error: Error }> = []
+
+  let index = 0
+  let active = 0
+  let resolve: (() => void) | null = null
+
+  const tick = async (): Promise<void> => {
+    while (active < concurrency && index < items.length) {
+      const item = items[index++]!
+      active++
+
+      ;(async () => {
+        try {
+          await processor(item)
+          succeeded.push(item)
+        } catch (err) {
+          failed.push({ item, error: err instanceof Error ? err : new Error(String(err)) })
+        } finally {
+          active--
+          if (resolve) {
+            const r = resolve
+            resolve = null
+            r()
+          }
+        }
+      })().catch(() => undefined)
+    }
+  }
+
+  await tick()
+
+  while (active > 0 || index < items.length) {
+    await new Promise<void>(r => { resolve = r })
+    await tick()
+  }
+
+  return { succeeded, failed }
+}
+
+/**
+ * Retry a function with exponential backoff.
+ * Only retries if the error is retryable (isRetryable === true for ApiError).
+ */
+export async function retryWithBackoff<T>(
+  fn: () => Promise<T>,
+  opts: { maxAttempts?: number; label?: string } = {},
+): Promise<T> {
+  const maxAttempts = opts.maxAttempts ?? 4
+  const delays = [0, 1000, 4000, 16000]
+  const log = getLogger()
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      return await fn()
+    } catch (err) {
+      const isLast = attempt === maxAttempts
+      const retryable = isRetryableError(err)
+
+      if (isLast || !retryable) {
+        if (attempt > 1) {
+          log.warn({ attempt, label: opts.label, err }, 'Giving up after retries')
+        }
+        throw err
+      }
+
+      const delay = delays[attempt] ?? 16000
+      log.debug({ attempt, delay, label: opts.label }, 'Retrying after delay')
+      await sleep(delay)
+    }
+  }
+
+  // Unreachable — TypeScript needs this
+  throw new Error('Retry loop exhausted')
+}
+
+function isRetryableError(err: unknown): boolean {
+  if (err instanceof ApiError) return err.isRetryable
+  if (err instanceof Error && err.message.includes('429')) return true
+  return false
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}

package/src/sync/incremental.ts

@@ -0,0 +1,111 @@
+import fs from 'node:fs/promises'
+import crypto from 'node:crypto'
+import { syncStatePath } from '../storage/paths.js'
+import { writeFileAtomic } from '../storage/atomic.js'
+import { SyncStateSchema, type SyncState, type RecordingState } from './types.js'
+import type { PlaudRecording } from '../client/types.js'
+import { getLogger } from '../logger.js'
+
+export class IncrementalTracker {
+  private state: SyncState = {
+    schemaVersion: 1,
+    recordings: {},
+  }
+
+  async load(outDir: string): Promise<void> {
+    const filePath = syncStatePath(outDir)
+    try {
+      const raw = await fs.readFile(filePath, 'utf8')
+      const json = JSON.parse(raw)
+      const result = SyncStateSchema.safeParse(json)
+      if (result.success) {
+        this.state = result.data
+        getLogger().debug(
+          { recordingCount: Object.keys(this.state.recordings).length },
+          'Loaded sync state',
+        )
+      } else {
+        getLogger().warn({ issues: result.error.issues }, 'Sync state schema invalid — starting fresh')
+      }
+    } catch (err: unknown) {
+      if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
+        getLogger().warn({ err }, 'Failed to read sync state — starting fresh')
+      }
+    }
+  }
+
+  async persist(outDir: string): Promise<void> {
+    this.state.lastAttemptAt = new Date().toISOString()
+    await writeFileAtomic(syncStatePath(outDir), JSON.stringify(this.state, null, 2))
+  }
+
+  markSuccessfulSync(): void {
+    this.state.lastSuccessfulSyncAt = new Date().toISOString()
+  }
+
+  getSince(): Date | undefined {
+    if (!this.state.lastSuccessfulSyncAt) return undefined
+    return new Date(this.state.lastSuccessfulSyncAt)
+  }
+
+  needsDownload(recording: PlaudRecording): boolean {
+    const existing = this.state.recordings[recording.id]
+    if (!existing) return true
+
+    const newHash = this.computeContentHash(recording)
+    if (existing.contentHash !== newHash) return true
+
+    // Re-download if key files are missing
+    if (!existing.hasTranscript && recording.hasTranscript) return true
+    if (!existing.downloadedAt) return true
+
+    return false
+  }
+
+  computeContentHash(recording: PlaudRecording): string {
+    const key = JSON.stringify({
+      id: recording.id,
+      updatedAt: recording.updatedAt,
+      hasTranscript: recording.hasTranscript,
+      transcriptStatus: recording.transcriptStatus,
+      duration: recording.duration,
+      title: recording.title,
+    })
+    return crypto.createHash('sha256').update(key).digest('hex').slice(0, 16)
+  }
+
+  markComplete(
+    recordingId: string,
+    recordedAt: string,
+    opts: { hasAudio: boolean; hasTranscript: boolean; contentHash: string },
+  ): void {
+    this.state.recordings[recordingId] = {
+      recordedAt,
+      contentHash: opts.contentHash,
+      downloadedAt: new Date().toISOString(),
+      hasAudio: opts.hasAudio,
+      hasTranscript: opts.hasTranscript,
+      verified: false,
+    }
+  }
+
+  markVerified(recordingId: string): void {
+    const existing = this.state.recordings[recordingId]
+    if (existing) {
+      existing.verified = true
+      existing.verifiedAt = new Date().toISOString()
+    }
+  }
+
+  getRecordingState(recordingId: string): RecordingState | undefined {
+    return this.state.recordings[recordingId]
+  }
+
+  getAllRecordingIds(): string[] {
+    return Object.keys(this.state.recordings)
+  }
+
+  get lastSuccessfulSyncAt(): string | undefined {
+    return this.state.lastSuccessfulSyncAt
+  }
+}