@smartcat/sanity-plugin 1.0.0 → 1.1.0

package/src/import/import.test.ts

@@ -0,0 +1,346 @@
+import {describe, expect, it, vi} from 'vitest'
+import {runImportDownload, type SanityLikeClient, type SmartcatImportClient} from './index'
+
+function fakeSanity(project: unknown) {
+  const sets: Record<string, unknown>[] = []
+  const client: SanityLikeClient = {
+    fetch: vi.fn(async () => project) as never,
+    patch: () => {
+      const attrs: Record<string, unknown> = {}
+      const p = {
+        set: (a: Record<string, unknown>) => {
+          Object.assign(attrs, a)
+          return p
+        },
+        commit: async () => {
+          sets.push(attrs)
+          return {}
+        },
+      }
+      return p
+    },
+  }
+  return {client, sets}
+}
+
+// One source doc ("doc1abcd…"): fr fully confirmed, de partially confirmed,
+// es with nothing confirmed yet.
+const PROJECT = {
+  _id: 'p1',
+  smartcatProjectId: 'sc-1',
+  items: [{doc: {_id: 'doc1abcd-eeee', title: 'About'}}],
+  progress: [],
+}
+const SC_PROJECT = {
+  id: 'sc-1',
+  workflowStages: [{id: 'st-tr', stageType: 'translation'}, {id: 'st-ed', stageType: 'editing'}],
+  documents: [
+    {id: 'g_12', name: 'About-doc1abcd', targetLanguage: 'fr', workflowStages: [{id: 'st-tr', progress: 100}, {id: 'st-ed', progress: 100}]},
+    {id: 'g_10', name: 'About-doc1abcd', targetLanguage: 'de', workflowStages: [{id: 'st-tr', progress: 100}, {id: 'st-ed', progress: 0}]},
+    {id: 'g_8', name: 'About-doc1abcd', targetLanguage: 'es', workflowStages: [{id: 'st-tr', progress: 0}, {id: 'st-ed', progress: 0}]},
+  ],
+}
+
+describe('runImportDownload', () => {
+  it('downloads targets with confirmed content (any stage) and skips the unconfirmed', async () => {
+    const {client: sanity, sets} = fakeSanity(PROJECT)
+    const smartcat: SmartcatImportClient = {
+      getProject: async () => SC_PROJECT,
+      exportDocument: vi.fn(async (id: string) => `{"id":"${id}"}`),
+    }
+
+    const result = await runImportDownload({sanity, smartcat, projectId: 'p1'})
+    // fr (complete) + de (partially confirmed) downloaded; es (nothing confirmed) skipped.
+    expect(result).toEqual({downloaded: 2, skipped: 1, remaining: 0, alreadyImported: 0})
+
+    const set = sets.at(-1)!
+    expect(set.status).toBe('downloaded')
+    const inbox = set.inbox as {targetLanguage: string; sourceDocId: string; locjson: string}[]
+    expect(inbox.map((i) => i.targetLanguage).sort()).toEqual(['de', 'fr'])
+    // Progress is refreshed for every target (sorted by language: de, es, fr).
+    const progress = set.progress as {targets: {language: string; complete: boolean}[]}[]
+    expect(progress[0].targets.map((t) => `${t.language}:${t.complete}`)).toEqual(['de:false', 'es:false', 'fr:true'])
+  })
+
+  it('uses one batched export per language and correlates files by x-sanity metadata', async () => {
+    // Two source documents, both complete in fr — one batch of two.
+    const project = {
+      _id: 'p1',
+      smartcatProjectId: 'sc-1',
+      items: [
+        {doc: {_id: 'doc1abcd-eeee', title: 'About'}},
+        {doc: {_id: 'doc2wxyz-ffff', title: 'Home'}},
+      ],
+      progress: [],
+    }
+    const scProject = {
+      id: 'sc-1',
+      workflowStages: [{id: 'st-tr', stageType: 'translation'}],
+      documents: [
+        {id: 'g_12', name: 'About-doc1abcd', targetLanguage: 'fr', workflowStages: [{id: 'st-tr', progress: 100}]},
+        {id: 'h_12', name: 'Home-doc2wxyz', targetLanguage: 'fr', workflowStages: [{id: 'st-tr', progress: 100}]},
+      ],
+    }
+    const locjsonFor = (docId: string) =>
+      JSON.stringify({properties: {'x-sanity': {documentId: docId}}, units: []})
+
+    const {client: sanity, sets} = fakeSanity(project)
+    const exportDocument = vi.fn(async () => '{}')
+    const exportDocumentsBatch = vi.fn(async () => [
+      // Archive order differs from request order on purpose.
+      {filename: 'b.locjson', content: locjsonFor('doc2wxyz-ffff')},
+      {filename: 'a.locjson', content: locjsonFor('doc1abcd-eeee')},
+    ])
+    const smartcat: SmartcatImportClient = {getProject: async () => scProject, exportDocument, exportDocumentsBatch}
+
+    const lines: string[] = []
+    const result = await runImportDownload({sanity, smartcat, projectId: 'p1', onLog: (l) => lines.push(`${l.level}|${l.message}`)})
+    expect(result).toEqual({downloaded: 2, skipped: 0, remaining: 0, alreadyImported: 0})
+    expect(exportDocumentsBatch).toHaveBeenCalledTimes(1)
+    expect(exportDocumentsBatch).toHaveBeenCalledWith(['g_12', 'h_12'])
+    expect(exportDocument).not.toHaveBeenCalled()
+
+    // The run narrates its batching so a degradation can never be silent.
+    expect(lines.some((l) => l.includes('Downloading 2 target(s) in 1 batch(es) of up to 50, 1 in parallel'))).toBe(true)
+    expect(lines.some((l) => l.includes('Batch 1/1 (fr): 2 doc(s) in'))).toBe(true)
+    expect(lines.some((l) => l.includes('Download phase:') && l.includes('0 failed, 0 per-document fallback'))).toBe(true)
+
+    const inbox = (sets.at(-1)!.inbox as {sourceDocId: string; locjson: string}[])
+    expect(inbox.find((i) => i.sourceDocId === 'doc1abcd-eeee')?.locjson).toBe(locjsonFor('doc1abcd-eeee'))
+    expect(inbox.find((i) => i.sourceDocId === 'doc2wxyz-ffff')?.locjson).toBe(locjsonFor('doc2wxyz-ffff'))
+  })
+
+  it('falls back to per-document export when the batch call fails or misses a file', async () => {
+    const project = {
+      _id: 'p1',
+      smartcatProjectId: 'sc-1',
+      items: [
+        {doc: {_id: 'doc1abcd-eeee', title: 'About'}},
+        {doc: {_id: 'doc2wxyz-ffff', title: 'Home'}},
+      ],
+      progress: [],
+    }
+    const scProject = {
+      id: 'sc-1',
+      workflowStages: [{id: 'st-tr', stageType: 'translation'}],
+      documents: [
+        {id: 'g_12', name: 'About-doc1abcd', targetLanguage: 'fr', workflowStages: [{id: 'st-tr', progress: 100}]},
+        {id: 'h_12', name: 'Home-doc2wxyz', targetLanguage: 'fr', workflowStages: [{id: 'st-tr', progress: 100}]},
+      ],
+    }
+    const {client: sanity} = fakeSanity(project)
+
+    // Whole batch fails → both documents fall back to single exports.
+    const failing: SmartcatImportClient = {
+      getProject: async () => scProject,
+      exportDocument: vi.fn(async (id: string) => `{"id":"${id}"}`),
+      exportDocumentsBatch: vi.fn(async () => {
+        throw new Error('500 from Smartcat')
+      }),
+    }
+    const failed = await runImportDownload({sanity, smartcat: failing, projectId: 'p1'})
+    expect(failed.downloaded).toBe(2)
+    expect(failing.exportDocument).toHaveBeenCalledTimes(2)
+
+    // Batch succeeds but covers only one document → the other falls back.
+    const partial: SmartcatImportClient = {
+      getProject: async () => scProject,
+      exportDocument: vi.fn(async (id: string) => `{"id":"${id}"}`),
+      exportDocumentsBatch: vi.fn(async () => [
+        {filename: 'a.locjson', content: JSON.stringify({properties: {'x-sanity': {documentId: 'doc1abcd-eeee'}}})},
+      ]),
+    }
+    const partialLines: string[] = []
+    const recovered = await runImportDownload({sanity, smartcat: partial, projectId: 'p1', onLog: (l) => partialLines.push(l.message)})
+    expect(recovered.downloaded).toBe(2)
+    expect(partial.exportDocument).toHaveBeenCalledTimes(1)
+    expect(partial.exportDocument).toHaveBeenCalledWith('h_12')
+    // Partial archive coverage is called out, not silently patched over.
+    expect(partialLines.some((m) => m.includes('1 of 2 file(s) missing from the batch archive'))).toBe(true)
+    expect(partialLines.some((m) => m.includes('1 per-document fallback download(s)'))).toBe(true)
+  })
+
+  it('skips already-imported targets and defers overflow beyond the inbox byte cap', async () => {
+    const bigLocjson = JSON.stringify({properties: {'x-sanity': {documentId: 'doc1abcd-eeee'}}, pad: 'x'.repeat(600)})
+    const project = {
+      _id: 'p1',
+      smartcatProjectId: 'sc-1',
+      items: [
+        {doc: {_id: 'doc1abcd-eeee', title: 'About'}},
+        {doc: {_id: 'doc2wxyz-ffff', title: 'Home'}},
+        {doc: {_id: 'doc3qrst-gggg', title: 'FAQ'}},
+      ],
+      // doc3 was already imported in a previous run.
+      progress: [
+        {
+          _key: 'doc3qrst-gggg__doc',
+          sourceDocId: 'doc3qrst-gggg',
+          targets: [{_key: 'k3', language: 'fr', smartcatDocumentId: 'i_12', stages: [], complete: true, imported: true}],
+        },
+      ],
+    }
+    const scProject = {
+      id: 'sc-1',
+      workflowStages: [{id: 'st-tr', stageType: 'translation'}],
+      documents: [
+        {id: 'g_12', name: 'About-doc1abcd', targetLanguage: 'fr', workflowStages: [{id: 'st-tr', progress: 100}]},
+        {id: 'h_12', name: 'Home-doc2wxyz', targetLanguage: 'fr', workflowStages: [{id: 'st-tr', progress: 100}]},
+        {id: 'i_12', name: 'FAQ-doc3qrst', targetLanguage: 'fr', workflowStages: [{id: 'st-tr', progress: 100}]},
+      ],
+    }
+    const {client: sanity, sets} = fakeSanity(project)
+    const smartcat: SmartcatImportClient = {
+      getProject: async () => scProject,
+      exportDocument: vi.fn(async () => bigLocjson),
+    }
+
+    // batchSize 1 → one download per batch; the first fills the tiny cap, the
+    // second is deferred; the already-imported third is skipped outright.
+    // concurrency 1 makes the soft cap deterministic (parallel workers may
+    // claim a batch before an in-flight download's bytes are counted).
+    const result = await runImportDownload({sanity, smartcat, projectId: 'p1', batchSize: 1, maxInboxBytes: 100, concurrency: 1})
+    expect(result).toEqual({downloaded: 1, skipped: 0, remaining: 1, alreadyImported: 1})
+    expect((sets.at(-1)!.inbox as unknown[]).length).toBe(1)
+    expect(sets.at(-1)!.status).toBe('downloaded')
+    expect(sets.at(-1)!.importRemaining).toBe(1)
+  })
+
+  it('runs up to 4 batch downloads in parallel', async () => {
+    const DOCS = 8
+    const items = Array.from({length: DOCS}, (_, i) => ({doc: {_id: `d${i}00000-aaaa`, title: `Doc ${i}`}}))
+    const documents = items.map(({doc}, i) => ({
+      id: `f${i}_1`,
+      name: `Doc-${doc._id.slice(0, 8)}`,
+      targetLanguage: 'fr',
+      workflowStages: [{id: 'st-tr', progress: 100}],
+    }))
+    const {client: sanity} = fakeSanity({_id: 'p1', smartcatProjectId: 'sc-1', items, progress: []})
+
+    let inFlight = 0
+    let maxInFlight = 0
+    const smartcat: SmartcatImportClient = {
+      getProject: async () => ({id: 'sc-1', workflowStages: [{id: 'st-tr', stageType: 'translation'}], documents}),
+      // batchSize 1 → every batch degrades to a single export; measure overlap.
+      exportDocument: vi.fn(async () => {
+        inFlight++
+        maxInFlight = Math.max(maxInFlight, inFlight)
+        await new Promise((r) => setTimeout(r, 5))
+        inFlight--
+        return '{}'
+      }),
+    }
+
+    const result = await runImportDownload({sanity, smartcat, projectId: 'p1', batchSize: 1})
+    expect(result.downloaded).toBe(8)
+    expect(maxInFlight).toBeGreaterThanOrEqual(2) // genuinely parallel…
+    expect(maxInFlight).toBeLessThanOrEqual(4) // …but never beyond the pool size
+  })
+
+  it('defers batches once the time budget is exhausted instead of overrunning the Function timeout', async () => {
+    const items = [
+      {doc: {_id: 'doc1abcd-eeee', title: 'About'}},
+      {doc: {_id: 'doc2wxyz-ffff', title: 'Home'}},
+    ]
+    const documents = [
+      {id: 'g_12', name: 'About-doc1abcd', targetLanguage: 'fr', workflowStages: [{id: 'st-tr', progress: 100}]},
+      {id: 'h_12', name: 'Home-doc2wxyz', targetLanguage: 'fr', workflowStages: [{id: 'st-tr', progress: 100}]},
+    ]
+    const {client: sanity, sets} = fakeSanity({_id: 'p1', smartcatProjectId: 'sc-1', items, progress: []})
+    const smartcat: SmartcatImportClient = {
+      getProject: async () => ({id: 'sc-1', workflowStages: [{id: 'st-tr', stageType: 'translation'}], documents}),
+      exportDocument: vi.fn(async () => '{}'),
+    }
+
+    // Clock: budget of 100ms; the first batch is claimed at t=0, the second at t=200.
+    const ticks = [0, 0, 200, 200, 200]
+    const nowMs = () => ticks.shift() ?? 200
+    const result = await runImportDownload({
+      sanity, smartcat, projectId: 'p1', batchSize: 1, concurrency: 1, timeBudgetMs: 100, nowMs,
+    })
+    expect(result).toEqual({downloaded: 1, skipped: 0, remaining: 1, alreadyImported: 0})
+    // The run still exits cleanly: status "downloaded" + the deferred count for auto-continue.
+    expect(sets.at(-1)!.status).toBe('downloaded')
+    expect(sets.at(-1)!.importRemaining).toBe(1)
+  })
+
+  it('handles a Huel-sized project (224 docs × 10 languages = 2,240 targets) in ~50 batched calls', async () => {
+    const LANGS = ['de', 'fr', 'es', 'it', 'da', 'fi', 'nl', 'pl', 'pt', 'sv']
+    const DOCS = 224
+    const itemId = (i: number) => `i${String(i).padStart(7, '0')}-aaaa`
+
+    const items = Array.from({length: DOCS}, (_, i) => ({doc: {_id: itemId(i), title: `Doc ${i}`}}))
+    const documents = items.flatMap(({doc}, i) =>
+      LANGS.map((lang, l) => ({
+        id: `f${i}_${l}`,
+        name: `Doc-${doc._id.slice(0, 8)}`,
+        targetLanguage: lang,
+        workflowStages: [{id: 'st-tr', progress: 100}],
+      })),
+    )
+    const sourceDocByScId = new Map<string, string>(
+      items.flatMap(({doc}, i) => LANGS.map((_, l) => [`f${i}_${l}`, doc._id])),
+    )
+    const locjsonFor = (scId: string) =>
+      JSON.stringify({properties: {'x-sanity': {documentId: sourceDocByScId.get(scId)}}, units: []})
+
+    let project: Record<string, unknown> = {_id: 'p1', smartcatProjectId: 'sc-1', items, progress: []}
+    const sets: Record<string, unknown>[] = []
+    const sanity: SanityLikeClient = {
+      fetch: (async () => project) as never,
+      patch: () => {
+        const attrs: Record<string, unknown> = {}
+        const p = {
+          set: (a: Record<string, unknown>) => (Object.assign(attrs, a), p),
+          commit: async () => (sets.push(attrs), {}),
+        }
+        return p
+      },
+    }
+    const exportDocumentsBatch = vi.fn(async (ids: string[]) => ids.map((id) => ({filename: `${id}.locjson`, content: locjsonFor(id)})))
+    const exportDocument = vi.fn(async (id: string) => locjsonFor(id))
+    const smartcat: SmartcatImportClient = {
+      getProject: async () => ({id: 'sc-1', workflowStages: [{id: 'st-tr', stageType: 'translation'}], documents}),
+      exportDocument,
+      exportDocumentsBatch,
+    }
+
+    // Payloads are small, so the default inbox cap fits the whole project in one
+    // run — and 2,240 targets cost ~50 Smartcat tasks instead of 2,240.
+    const result = await runImportDownload({sanity, smartcat, projectId: 'p1'})
+    expect(result).toEqual({downloaded: 2240, skipped: 0, remaining: 0, alreadyImported: 0})
+    expect(exportDocumentsBatch).toHaveBeenCalledTimes(50) // 10 languages × ceil(224/50)
+    expect(exportDocument).not.toHaveBeenCalled()
+
+    // Capped runs converge: simulate the browser apply (mark downloaded targets
+    // imported) between runs and re-import until nothing is left.
+    project = {...project, progress: []}
+    sets.length = 0
+    exportDocumentsBatch.mockClear()
+    let imported = 0
+    for (let run = 0; run < 20; run++) {
+      const r = await runImportDownload({sanity, smartcat, projectId: 'p1', maxInboxBytes: 30_000})
+      expect(r.alreadyImported).toBe(imported)
+      imported += r.downloaded
+      const set = sets.at(-1)!
+      const inboxKeys = new Set((set.inbox as {_key: string}[]).map((i) => i._key))
+      const progress = (set.progress as {targets: {_key: string; imported: boolean}[]}[]).map((doc) => ({
+        ...doc,
+        targets: doc.targets.map((t) => (inboxKeys.has(t._key) ? {...t, imported: true} : t)),
+      }))
+      project = {...project, progress}
+      if (r.remaining === 0) break
+      expect(r.downloaded).toBeGreaterThan(0) // every run makes forward progress
+    }
+    expect(imported).toBe(2240)
+  })
+
+  it('errors when the project has no smartcatProjectId', async () => {
+    const {client: sanity, sets} = fakeSanity({_id: 'p1'})
+    const smartcat: SmartcatImportClient = {
+      getProject: async () => ({id: 'x'}),
+      exportDocument: async () => '',
+    }
+    await expect(runImportDownload({sanity, smartcat, projectId: 'p1'})).rejects.toThrow(/smartcatProjectId/)
+    expect(sets.at(-1)?.status).toBe('error')
+  })
+})