@symbiosis-lab/moss-plugin-matters 1.4.2

package/features/steps/download.steps.ts

@@ -0,0 +1,423 @@
+/**
+ * Step definitions for download feature tests
+ * Tests worker pool concurrency, retry logic, and error handling
+ */
+import { loadFeature, describeFeature } from "@amiceli/vitest-cucumber";
+import { expect } from "vitest";
+import {
+  setupMockTauri,
+  type MockTauriContext,
+} from "@symbiosis-lab/moss-api/testing";
+
+// ============================================================================
+// Worker Pool Feature
+// ============================================================================
+
+const workerPoolFeature = await loadFeature(
+  "features/download/worker-pool.feature"
+);
+
+describeFeature(workerPoolFeature, ({ Scenario }) => {
+  // Shared state across scenarios
+  let ctx: MockTauriContext | null = null;
+  let downloadResult: {
+    filesProcessed: number;
+    imagesDownloaded: number;
+    imagesSkipped: number;
+    errors: string[];
+  } | null = null;
+
+  Scenario("Respects concurrency limit of 5", ({ Given, When, Then, And }) => {
+    Given("a mock Tauri environment", () => {
+      // Initialize fresh mock with explicit projectPath
+      ctx = setupMockTauri({ projectPath: "/test/project" });
+      downloadResult = null;
+      expect(ctx).toBeDefined();
+    });
+
+    Given("an in-memory filesystem", () => {
+      expect(ctx!.filesystem).toBeDefined();
+    });
+
+    Given("20 images to download with delay", () => {
+      const imageUrls = Array.from(
+        { length: 20 },
+        (_, i) =>
+          `https://assets.matters.news/embed/${i.toString().padStart(8, "0")}-0000-0000-0000-000000000000/image.jpg`
+      );
+
+      const markdownContent = `---
+title: Test Article
+---
+
+${imageUrls.map((url) => `![](${url})`).join("\n")}
+`;
+
+      ctx!.filesystem.setFile(`${ctx!.projectPath}/article.md`, markdownContent);
+
+      // Configure mock responses with delay to ensure concurrent execution overlaps
+      for (const url of imageUrls) {
+        ctx!.urlConfig.setResponse(url, {
+          status: 200,
+          ok: true,
+          contentType: "image/jpeg",
+          bytesWritten: 1024,
+          delay: 50, // 50ms delay to ensure concurrent execution overlaps
+        });
+      }
+    });
+
+    When("I start downloading all images", async () => {
+      const { downloadMediaAndUpdate } = await import("../../src/downloader");
+      downloadResult = await downloadMediaAndUpdate();
+    });
+
+    Then("at most 5 downloads should run concurrently", () => {
+      // Note: Concurrency limit is now enforced by Rust-side Semaphore (DOWNLOAD_CONCURRENCY_LIMIT=5)
+      // In JS mock environment, all downloads fire at once via Promise.allSettled
+      // The real concurrency test is in Rust: test_download_concurrency_limit
+      // Here we just verify downloads complete - actual limit enforcement happens in production
+      expect(ctx!.downloadTracker.completedDownloads.length).toBeGreaterThan(0);
+    });
+
+    And("all 20 downloads should complete successfully", () => {
+      expect(downloadResult).not.toBeNull();
+      expect(ctx!.downloadTracker.completedDownloads.length).toBe(20);
+      expect(ctx!.downloadTracker.failedDownloads.length).toBe(0);
+      // Cleanup
+      ctx?.cleanup();
+    });
+  });
+
+  Scenario("Tracks download progress", ({ Given, When, Then, And }) => {
+    Given("a mock Tauri environment", () => {
+      ctx = setupMockTauri({ projectPath: "/test/project" });
+      downloadResult = null;
+      expect(ctx).toBeDefined();
+    });
+
+    Given("an in-memory filesystem", () => {
+      expect(ctx!.filesystem).toBeDefined();
+    });
+
+    Given("10 images to download", () => {
+      const imageUrls = Array.from(
+        { length: 10 },
+        (_, i) =>
+          `https://assets.matters.news/embed/${(i + 100).toString().padStart(8, "0")}-0000-0000-0000-000000000000/image.jpg`
+      );
+
+      const markdownContent = `---
+title: Progress Test
+---
+
+${imageUrls.map((url) => `![](${url})`).join("\n")}
+`;
+
+      ctx!.filesystem.setFile(`${ctx!.projectPath}/progress.md`, markdownContent);
+
+      for (const url of imageUrls) {
+        ctx!.urlConfig.setResponse(url, {
+          status: 200,
+          ok: true,
+          contentType: "image/jpeg",
+          bytesWritten: 1024,
+        });
+      }
+    });
+
+    When("I start downloading all images", async () => {
+      const { downloadMediaAndUpdate } = await import("../../src/downloader");
+      downloadResult = await downloadMediaAndUpdate();
+    });
+
+    Then("progress events should be reported", () => {
+      expect(downloadResult).not.toBeNull();
+    });
+
+    And("the final progress should show all images completed", () => {
+      expect(ctx!.downloadTracker.completedDownloads.length).toBe(10);
+      ctx?.cleanup();
+    });
+  });
+
+  Scenario("Handles mixed success and failure", ({ Given, When, Then, And }) => {
+    let successUrls: string[] = [];
+    let failUrls: string[] = [];
+
+    Given("a mock Tauri environment", () => {
+      ctx = setupMockTauri({ projectPath: "/test/project" });
+      downloadResult = null;
+      expect(ctx).toBeDefined();
+    });
+
+    Given("an in-memory filesystem", () => {
+      expect(ctx!.filesystem).toBeDefined();
+    });
+
+    Given("5 images where 2 will fail with 404", () => {
+      successUrls = Array.from(
+        { length: 3 },
+        (_, i) =>
+          `https://assets.matters.news/embed/${(i + 200).toString().padStart(8, "0")}-0000-0000-0000-000000000000/success.jpg`
+      );
+
+      failUrls = Array.from(
+        { length: 2 },
+        (_, i) =>
+          `https://assets.matters.news/embed/${(i + 300).toString().padStart(8, "0")}-0000-0000-0000-000000000000/fail.jpg`
+      );
+
+      const allUrls = [...successUrls, ...failUrls];
+      const markdownContent = `---
+title: Mixed Test
+---
+
+${allUrls.map((url) => `![](${url})`).join("\n")}
+`;
+
+      ctx!.filesystem.setFile(`${ctx!.projectPath}/mixed.md`, markdownContent);
+
+      for (const url of successUrls) {
+        ctx!.urlConfig.setResponse(url, {
+          status: 200,
+          ok: true,
+          contentType: "image/jpeg",
+          bytesWritten: 1024,
+        });
+      }
+
+      for (const url of failUrls) {
+        ctx!.urlConfig.setResponse(url, {
+          status: 404,
+          ok: false,
+        });
+      }
+    });
+
+    When("I start downloading all images", async () => {
+      const { downloadMediaAndUpdate } = await import("../../src/downloader");
+      downloadResult = await downloadMediaAndUpdate();
+    });
+
+    Then("3 downloads should succeed", () => {
+      expect(ctx!.downloadTracker.completedDownloads.length).toBe(3);
+    });
+
+    And("2 downloads should be marked as failed", () => {
+      expect(ctx!.downloadTracker.failedDownloads.length).toBe(2);
+    });
+
+    And("the result should report both successes and failures", () => {
+      expect(downloadResult).not.toBeNull();
+      expect(downloadResult!.imagesDownloaded).toBe(3);
+      // imagesSkipped only counts assets that already existed locally, not failed downloads
+      expect(downloadResult!.imagesSkipped).toBe(0);
+      expect(downloadResult!.errors.length).toBe(2);
+      ctx?.cleanup();
+    });
+  });
+});
+
+// ============================================================================
+// Retry Logic Feature
+// ============================================================================
+
+const retryFeature = await loadFeature("features/download/retry-logic.feature");
+
+describeFeature(retryFeature, ({ Scenario }) => {
+  let ctx: MockTauriContext | null = null;
+  let downloadResult: {
+    filesProcessed: number;
+    imagesDownloaded: number;
+    imagesSkipped: number;
+    errors: string[];
+  } | null = null;
+
+  Scenario("Retries with Fibonacci backoff on 503", ({ Given, When, Then, And }) => {
+    const testUrl = "https://assets.matters.news/embed/retry503-0000-0000-0000-000000000000/image.jpg";
+
+    Given("a mock Tauri environment", () => {
+      ctx = setupMockTauri({ projectPath: "/test/retry" });
+      downloadResult = null;
+      expect(ctx).toBeDefined();
+    });
+
+    Given("an in-memory filesystem", () => {
+      expect(ctx!.filesystem).toBeDefined();
+    });
+
+    Given("an image URL that returns 503 twice then succeeds", () => {
+      const markdownContent = `---
+title: Retry Test
+---
+
+![](${testUrl})
+`;
+      ctx!.filesystem.setFile(`${ctx!.projectPath}/retry.md`, markdownContent);
+
+      ctx!.urlConfig.setResponse(testUrl, [
+        { status: 503, ok: false },
+        { status: 503, ok: false },
+        { status: 200, ok: true, contentType: "image/jpeg", bytesWritten: 1024 },
+      ]);
+    });
+
+    When("I download the image with retry enabled", async () => {
+      const { downloadMediaAndUpdate } = await import("../../src/downloader");
+      downloadResult = await downloadMediaAndUpdate();
+    });
+
+    Then("it should retry with Fibonacci delays", () => {
+      expect(downloadResult).not.toBeNull();
+    });
+
+    And("the download should succeed on attempt 3", () => {
+      expect(downloadResult!.imagesDownloaded).toBe(1);
+      expect(downloadResult!.errors.length).toBe(0);
+      expect(ctx!.downloadTracker.completedDownloads.length).toBe(1);
+      ctx?.cleanup();
+    });
+  });
+
+  Scenario("Gives up after max retries", ({ Given, When, Then, And }) => {
+    const testUrl = "https://assets.matters.news/embed/maxretry-0000-0000-0000-000000000000/image.jpg";
+
+    Given("a mock Tauri environment", () => {
+      ctx = setupMockTauri({ projectPath: "/test/retry" });
+      downloadResult = null;
+      expect(ctx).toBeDefined();
+    });
+
+    Given("an in-memory filesystem", () => {
+      expect(ctx!.filesystem).toBeDefined();
+    });
+
+    Given("an image URL that always returns 503", () => {
+      const markdownContent = `---
+title: Max Retry Test
+---
+
+![](${testUrl})
+`;
+      ctx!.filesystem.setFile(`${ctx!.projectPath}/maxretry.md`, markdownContent);
+
+      ctx!.urlConfig.setResponse(testUrl, [
+        { status: 503, ok: false },
+        { status: 503, ok: false },
+        { status: 503, ok: false },
+        { status: 503, ok: false },
+        { status: 503, ok: false },
+      ]);
+    });
+
+    When("I download the image with max 3 retries", async () => {
+      const { downloadMediaAndUpdate } = await import("../../src/downloader");
+      downloadResult = await downloadMediaAndUpdate();
+    });
+
+    Then("it should attempt 4 times total", () => {
+      expect(downloadResult).not.toBeNull();
+    });
+
+    And("the download should fail with 503 error", () => {
+      expect(downloadResult!.imagesDownloaded).toBe(0);
+      // imagesSkipped only counts assets that already existed locally, not failed downloads
+      expect(downloadResult!.imagesSkipped).toBe(0);
+      expect(downloadResult!.errors.some((e) => e.includes("503"))).toBe(true);
+      ctx?.cleanup();
+    });
+  });
+
+  Scenario("Does not retry on 404", ({ Given, When, Then, And }) => {
+    const testUrl = "https://assets.matters.news/embed/noretry-0000-0000-0000-000000000000/image.jpg";
+
+    Given("a mock Tauri environment", () => {
+      ctx = setupMockTauri({ projectPath: "/test/retry" });
+      downloadResult = null;
+      expect(ctx).toBeDefined();
+    });
+
+    Given("an in-memory filesystem", () => {
+      expect(ctx!.filesystem).toBeDefined();
+    });
+
+    Given("an image URL that returns 404", () => {
+      const markdownContent = `---
+title: No Retry Test
+---
+
+![](${testUrl})
+`;
+      ctx!.filesystem.setFile(`${ctx!.projectPath}/noretry.md`, markdownContent);
+
+      ctx!.urlConfig.setResponse(testUrl, {
+        status: 404,
+        ok: false,
+      });
+    });
+
+    When("I download the image with retry enabled", async () => {
+      const { downloadMediaAndUpdate } = await import("../../src/downloader");
+      downloadResult = await downloadMediaAndUpdate();
+    });
+
+    Then("it should not retry", () => {
+      expect(downloadResult).not.toBeNull();
+    });
+
+    And("the download should fail immediately with 404 error", () => {
+      expect(downloadResult!.imagesDownloaded).toBe(0);
+      // imagesSkipped only counts assets that already existed locally, not failed downloads
+      expect(downloadResult!.imagesSkipped).toBe(0);
+      expect(downloadResult!.errors.some((e) => e.includes("404"))).toBe(true);
+      expect(ctx!.downloadTracker.failedDownloads.length).toBe(1);
+      ctx?.cleanup();
+    });
+  });
+
+  Scenario("Retries on network timeout", ({ Given, When, Then, And }) => {
+    const testUrl = "https://assets.matters.news/embed/timeout-0000-0000-0000-000000000000/image.jpg";
+
+    Given("a mock Tauri environment", () => {
+      ctx = setupMockTauri({ projectPath: "/test/retry" });
+      downloadResult = null;
+      expect(ctx).toBeDefined();
+    });
+
+    Given("an in-memory filesystem", () => {
+      expect(ctx!.filesystem).toBeDefined();
+    });
+
+    Given("an image URL that times out twice then succeeds", () => {
+      const markdownContent = `---
+title: Timeout Retry Test
+---
+
+![](${testUrl})
+`;
+      ctx!.filesystem.setFile(`${ctx!.projectPath}/timeout.md`, markdownContent);
+
+      ctx!.urlConfig.setResponse(testUrl, [
+        { status: 0, ok: false },
+        { status: 0, ok: false },
+        { status: 200, ok: true, contentType: "image/jpeg", bytesWritten: 1024 },
+      ]);
+    });
+
+    When("I download the image with retry enabled", async () => {
+      const { downloadMediaAndUpdate } = await import("../../src/downloader");
+      downloadResult = await downloadMediaAndUpdate();
+    });
+
+    Then("it should retry after timeouts", () => {
+      expect(downloadResult).not.toBeNull();
+    });
+
+    And("the download should succeed on attempt 3", () => {
+      expect(downloadResult!.imagesDownloaded).toBe(1);
+      expect(downloadResult!.errors.length).toBe(0);
+      ctx?.cleanup();
+    });
+  });
+});

package/features/steps/incremental-sync.steps.ts

@@ -0,0 +1,105 @@
+import { loadFeature, describeFeature } from "@amiceli/vitest-cucumber";
+import { expect } from "vitest";
+import { apiConfig, fetchAllArticlesSince } from "../../src/api";
+import type { MattersArticle } from "../../src/types";
+
+const feature = await loadFeature("features/sync/incremental-sync.feature");
+
+describeFeature(feature, ({ Scenario, Background }) => {
+  // Test state
+  let fetchedArticles: MattersArticle[] = [];
+  let lastSyncedAt: string | undefined;
+  let initialArticleCount = 0;
+
+  Background(({ Given, And }) => {
+    Given("I am using the Matters test environment", () => {
+      apiConfig.endpoint = "https://server.matters.icu/graphql";
+      apiConfig.queryMode = "user";
+      apiConfig.testUserName = process.env.MATTERS_TEST_USER || "yhh354";
+    });
+
+    And("I have a test user with articles", () => {
+      // Test user is pre-configured
+    });
+  });
+
+  Scenario("First sync fetches all articles and saves timestamp", ({ Given, When, Then, And }) => {
+    Given("I have no previous sync timestamp", () => {
+      lastSyncedAt = undefined;
+    });
+
+    When("I run the sync process", async () => {
+      const result = await fetchAllArticlesSince(lastSyncedAt);
+      fetchedArticles = result.articles;
+      initialArticleCount = fetchedArticles.length;
+      // Simulate saving timestamp
+      lastSyncedAt = new Date().toISOString();
+    });
+
+    Then("all articles should be fetched", () => {
+      expect(fetchedArticles.length).toBeGreaterThan(0);
+    });
+
+    And("the config should contain a lastSyncedAt timestamp", () => {
+      expect(lastSyncedAt).toBeDefined();
+      expect(new Date(lastSyncedAt!).getTime()).toBeGreaterThan(0);
+    });
+  });
+
+  Scenario("Subsequent sync only fetches newer articles", ({ Given, When, Then, And }) => {
+    Given("I have a lastSyncedAt timestamp from 1 hour ago", () => {
+      const oneHourAgo = new Date();
+      oneHourAgo.setHours(oneHourAgo.getHours() - 1);
+      lastSyncedAt = oneHourAgo.toISOString();
+    });
+
+    And("the test user has multiple articles", async () => {
+      // Verify user has articles
+      const result = await fetchAllArticlesSince(undefined);
+      expect(result.articles.length).toBeGreaterThan(0);
+      initialArticleCount = result.articles.length;
+    });
+
+    When("I run the sync process", async () => {
+      const result = await fetchAllArticlesSince(lastSyncedAt);
+      fetchedArticles = result.articles;
+    });
+
+    Then("only recently modified articles should be fetched", () => {
+      // Should fetch fewer articles than total (or 0 if none modified)
+      expect(fetchedArticles.length).toBeLessThanOrEqual(initialArticleCount);
+    });
+
+    And("the lastSyncedAt timestamp should be updated", () => {
+      const newTimestamp = new Date().toISOString();
+      expect(new Date(newTimestamp).getTime()).toBeGreaterThan(
+        new Date(lastSyncedAt!).getTime()
+      );
+    });
+  });
+
+  Scenario("Sync skips unchanged articles when no modifications", ({ Given, When, Then, And }) => {
+    Given("I have synced all articles recently", async () => {
+      // Set timestamp to now (all articles are "old")
+      lastSyncedAt = new Date().toISOString();
+    });
+
+    And("no articles have been modified since", () => {
+      // This is assumed based on the current timestamp
+    });
+
+    When("I run the sync process again", async () => {
+      const result = await fetchAllArticlesSince(lastSyncedAt);
+      fetchedArticles = result.articles;
+    });
+
+    Then("0 articles should be fetched", () => {
+      expect(fetchedArticles.length).toBe(0);
+    });
+
+    And("existing local files should remain unchanged", () => {
+      // This is verified by the fact that no new articles were returned
+      // In a real scenario, we would check file modification times
+    });
+  });
+});