@dxos/plugin-magazine 0.0.0

package/src/operations/sync-feed.test.ts

@@ -0,0 +1,146 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { describe, it } from '@effect/vitest';
+import * as Effect from 'effect/Effect';
+import { afterEach, beforeEach, vi } from 'vitest';
+
+import { Operation } from '@dxos/compute';
+import { Database, Feed, Filter, Ref, Tag } from '@dxos/echo';
+import { TestHelpers } from '@dxos/effect/testing';
+import { AssistantTestLayer } from '@dxos/functions-runtime/testing';
+import { EntityId } from '@dxos/keys';
+
+import { FeedOperation, Subscription } from '../types';
+import { MagazineOperationHandlerSet } from './index';
+
+EntityId.dangerouslyDisableRandomness();
+
+const TestLayer = AssistantTestLayer({
+  operationHandlers: MagazineOperationHandlerSet,
+  types: [Feed.Feed, Subscription.Subscription, Subscription.Post, Tag.Tag],
+  disableLlmMemoization: true,
+});
+
+/**
+ * Canned RSS XML returned by the stubbed global `fetch`. Two items with stable
+ * guids — mirrors what an upstream RSS feed serves on repeated polls when no
+ * new posts have been published.
+ */
+const RSS_XML = `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>Test Feed</title>
+    <description>A test feed</description>
+    <item>
+      <title>Post A</title>
+      <link>https://example.com/a</link>
+      <guid>post-a</guid>
+      <description>A body</description>
+      <pubDate>Wed, 01 Apr 2026 00:00:00 GMT</pubDate>
+    </item>
+    <item>
+      <title>Post B</title>
+      <link>https://example.com/b</link>
+      <guid>post-b</guid>
+      <description>B body</description>
+      <pubDate>Thu, 02 Apr 2026 00:00:00 GMT</pubDate>
+    </item>
+  </channel>
+</rss>`;
+
+/**
+ * End-to-end tests for the {@link FeedOperation.SyncFeed} operation against
+ * the full operation harness (Database + Feed services + handler registry).
+ *
+ * Stubs only the network boundary (`globalThis.fetch`) so the test exercises
+ * the real RSS parser, real ECHO writes, and the real operation runtime.
+ *
+ * The original bug: each sync re-appended every previously-seen post because
+ * the cursor filter `post.guid !== cursor` only excluded the single post whose
+ * guid matched the cursor — every older post passed through and was appended
+ * again. These tests pin the corrected, guid-deduped behavior.
+ */
+describe('SyncFeed', () => {
+  beforeEach(() => {
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(async () => new Response(RSS_XML, { headers: { 'content-type': 'application/xml' } })),
+    );
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+  });
+
+  it.effect(
+    're-syncing the same upstream response does not duplicate posts',
+    Effect.fnUntraced(
+      function* ({ expect }) {
+        const subscriptionFeed = yield* Database.add(
+          Subscription.makeSubscription({ name: 'feed', url: 'https://example.com/rss' }),
+        );
+        yield* Database.flush();
+        const echoFeed = yield* Database.load(subscriptionFeed.feed);
+
+        yield* Operation.invoke(FeedOperation.SyncFeed, { feed: Ref.make(subscriptionFeed) });
+        yield* Operation.invoke(FeedOperation.SyncFeed, { feed: Ref.make(subscriptionFeed) });
+
+        const items = yield* Feed.query(echoFeed, Filter.type(Subscription.Post)).run;
+        expect(items).toHaveLength(2);
+        const guids = items.map((post) => post.guid).sort();
+        expect(guids).toEqual(['post-a', 'post-b']);
+      },
+      Effect.provide(TestLayer),
+      TestHelpers.provideTestContext,
+    ),
+  );
+
+  it.effect(
+    'within-batch and link-fallback dedup: same link served twice in one response is appended once',
+    Effect.fnUntraced(
+      function* ({ expect }) {
+        // RSS items with NO `<guid>` element — the dedup key falls back to
+        // `link`. Two items share the same link, simulating a misbehaving
+        // upstream that serves the same post twice in one response.
+        const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>Duplicate Feed</title>
+    <item>
+      <title>First copy</title>
+      <link>https://example.com/shared</link>
+      <description>body</description>
+    </item>
+    <item>
+      <title>Second copy</title>
+      <link>https://example.com/shared</link>
+      <description>body</description>
+    </item>
+  </channel>
+</rss>`;
+        vi.stubGlobal(
+          'fetch',
+          vi.fn(async () => new Response(xml, { headers: { 'content-type': 'application/xml' } })),
+        );
+
+        const subscriptionFeed = yield* Database.add(
+          Subscription.makeSubscription({ name: 'feed', url: 'https://example.com/rss' }),
+        );
+        yield* Database.flush();
+        const echoFeed = yield* Database.load(subscriptionFeed.feed);
+
+        yield* Operation.invoke(FeedOperation.SyncFeed, { feed: Ref.make(subscriptionFeed) });
+        // Second sync also exercises the persisted-post dedup path for link-keyed items.
+        yield* Operation.invoke(FeedOperation.SyncFeed, { feed: Ref.make(subscriptionFeed) });
+
+        const items = yield* Feed.query(echoFeed, Filter.type(Subscription.Post)).run;
+        expect(items).toHaveLength(1);
+        expect(items[0].link).toBe('https://example.com/shared');
+      },
+      Effect.provide(TestLayer),
+      TestHelpers.provideTestContext,
+    ),
+  );
+});

package/src/operations/sync-feed.ts

@@ -0,0 +1,140 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Operation } from '@dxos/compute';
+import { Database, Feed, Filter, Obj, Ref } from '@dxos/echo';
+import { invariant } from '@dxos/invariant';
+
+import { FeedOperation, Subscription } from '../types';
+import { type FeedFetcher, browserCorsProxy, fetchRss, fetchStandardSite } from './sources';
+
+/** Stable dedup key for a {@link Subscription.Post}. Both fields are optional, but every current fetcher populates `guid` (RSS falls back to `link`, Standard.site uses the record AT-URI). */
+const postKey = (post: { guid?: string; link?: string }): string | undefined => post.guid ?? post.link;
+
+/** Resolves the appropriate fetcher for the given feed type. */
+const getFetcher = (type: Subscription.FeedType | undefined): FeedFetcher => {
+  switch (type) {
+    case 'standard-site':
+      return fetchStandardSite;
+    case 'rss':
+    default:
+      return fetchRss;
+  }
+};
+
+const handler: Operation.WithHandler<typeof FeedOperation.SyncFeed> = FeedOperation.SyncFeed.pipe(
+  Operation.withHandler(
+    Effect.fnUntraced(function* ({ feed }) {
+      const subscriptionFeed = yield* Database.load(feed);
+      const url = subscriptionFeed.url;
+      invariant(url, 'Feed URL is required.');
+      const echoFeed = yield* Database.load(subscriptionFeed.feed);
+      invariant(echoFeed, 'Backing ECHO feed not found.');
+      invariant(Feed.getQueueUri(echoFeed), 'Feed not stored in a space.');
+
+      const fetcher = getFetcher(subscriptionFeed.type);
+      const { feed: feedMeta, posts } = yield* fetcher(url, { corsProxy: browserCorsProxy() });
+
+      // Dedup against existing posts already in the backing queue. The
+      // `cursor` field on the subscription was previously used as a single-guid
+      // filter, but RSS feeds re-serve the same items across polls — filtering
+      // out only the cursor's own guid let every other previously-synced post
+      // re-pass the filter and re-append on each sync. Key-based dedup against
+      // the queue is the source of truth and tolerant of out-of-order or
+      // rotated-off-the-window upstream responses.
+      //
+      // The dedup key is `guid ?? link`: `Post.guid` is optional in the schema,
+      // and while every current fetcher populates it (RSS falls back to `link`,
+      // Standard.site uses the record AT-URI), a future fetcher or malformed item
+      // could omit it. Falling back to `link` keeps such items dedup-able. Items
+      // with neither field cannot be deduplicated and will sync as new every
+      // time — that's an upstream data quality issue with no clean recovery.
+      //
+      // Pull the backing queue before reading existing posts: this handler runs in a
+      // freshly-spawned process whose local queue replica may not have the blocks yet. Without
+      // this, the query returns an empty set, `seenKeys` is empty, and every fetched item
+      // re-appends — duplicating the whole feed on a cold sync.
+      yield* Feed.sync(echoFeed, { shouldPush: false });
+      const existing = yield* Feed.query(echoFeed, Filter.type(Subscription.Post)).run;
+      const seenKeys = new Set<string>();
+      for (const post of existing) {
+        const key = postKey(post);
+        if (key) {
+          seenKeys.add(key);
+        }
+      }
+      const newPosts: typeof posts = [];
+      for (const post of posts) {
+        const key = postKey(post);
+        // Within-batch dedup: if the upstream serves the same item twice in one
+        // response, only the first occurrence is appended.
+        if (key && seenKeys.has(key)) {
+          continue;
+        }
+        if (key) {
+          seenKeys.add(key);
+        }
+        newPosts.push(post);
+      }
+
+      // Append new posts to the ECHO feed.
+      // NOTE: The `Subscription.Subscription.keep` bound is currently NOT enforced
+      // here via `Feed.remove()`. Doing so wipes the underlying queue's
+      // `_objectCache` for the deleted posts, but those same Post objects
+      // persist in `space.db` (they were added there when first curated
+      // into a Magazine via `createRef` → `database.add`). On the next
+      // sync/curate, `Feed.query(...).run` returns fresh proxies for the
+      // kept items, and any magazine refs to *deleted* posts now
+      // reference proxies whose `_internals.database` link is unset —
+      // `createRef` then tries to re-add them, hitting the
+      // `!_objects.has(core.id)` invariant in `EntityManager.addCore`.
+      // Until feed/db lifecycle is reworked we leave the feed unbounded;
+      // the `Magazine.keep` bound (enforced in
+      // `MagazineArticle.handleCurate`) prevents the visible list from
+      // growing unboundedly.
+      if (newPosts.length > 0) {
+        const feedRef = Ref.make(subscriptionFeed);
+        const postObjects = newPosts.map((post) =>
+          Obj.make(Subscription.Post, {
+            source: feedRef,
+            title: post.title,
+            link: post.link,
+            description: post.description,
+            // Persist the body so Standard.site articles render offline (see `load-post-content`);
+            // RSS `content:encoded` rides along but its fetch path is unchanged.
+            content: post.content,
+            author: post.author,
+            published: post.published,
+            guid: post.guid,
+          }),
+        );
+        yield* Feed.append(echoFeed, postObjects);
+
+        // Advance cursor to the newest post.
+        const newestGuid = posts[0]?.guid;
+        if (newestGuid) {
+          Obj.update(subscriptionFeed, (subscriptionFeed) => {
+            subscriptionFeed.cursor = newestGuid;
+          });
+        }
+
+        // Update feed metadata from channel if not already set.
+        if (feedMeta.name && !subscriptionFeed.name) {
+          Obj.update(subscriptionFeed, (subscriptionFeed) => {
+            subscriptionFeed.name = feedMeta.name;
+          });
+        }
+        if (feedMeta.description && !subscriptionFeed.description) {
+          Obj.update(subscriptionFeed, (subscriptionFeed) => {
+            subscriptionFeed.description = feedMeta.description;
+          });
+        }
+      }
+    }),
+  ),
+);
+
+export default handler;

package/src/operations/util.test.ts

@@ -0,0 +1,67 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { describe, expect, test } from 'vitest';
+
+import { Obj } from '@dxos/echo';
+
+import { Subscription } from '../types';
+import { partitionByKeepBound } from './util';
+
+const makePost = (props: { title: string; published?: string }): Subscription.Post =>
+  Obj.make(Subscription.Post, { title: props.title, published: props.published });
+
+describe('partitionByKeepBound', () => {
+  test('keeps the newest N non-starred posts and drops the rest', () => {
+    const posts = [
+      makePost({ title: 'A', published: '2026-04-01T00:00:00Z' }),
+      makePost({ title: 'B', published: '2026-04-02T00:00:00Z' }),
+      makePost({ title: 'C', published: '2026-04-03T00:00:00Z' }),
+      makePost({ title: 'D', published: '2026-04-04T00:00:00Z' }),
+      makePost({ title: 'E', published: '2026-04-05T00:00:00Z' }),
+    ];
+    const { kept, dropped } = partitionByKeepBound(posts, 2, () => false);
+    // Keep newest two (E, D); drop the older three.
+    expect(kept.map((post) => post.title).sort()).toEqual(['D', 'E']);
+    expect(dropped.map((post) => post.title).sort()).toEqual(['A', 'B', 'C']);
+  });
+
+  test('preserves starred posts beyond the keep bound', () => {
+    const posts = [
+      makePost({ title: 'A-old-starred', published: '2026-04-01T00:00:00Z' }),
+      makePost({ title: 'B', published: '2026-04-02T00:00:00Z' }),
+      makePost({ title: 'C', published: '2026-04-03T00:00:00Z' }),
+      makePost({ title: 'D', published: '2026-04-04T00:00:00Z' }),
+      makePost({ title: 'E', published: '2026-04-05T00:00:00Z' }),
+    ];
+    const isStarred = (post: Subscription.Post) => post.title === 'A-old-starred';
+    const { kept, dropped } = partitionByKeepBound(posts, 2, isStarred);
+    // Starred A always kept; newest non-starred E and D fill the bound; B and C dropped.
+    expect(kept.map((post) => post.title).sort()).toEqual(['A-old-starred', 'D', 'E']);
+    expect(dropped.map((post) => post.title).sort()).toEqual(['B', 'C']);
+  });
+
+  test('falls back gracefully when posts have no published date', () => {
+    const posts = [
+      makePost({ title: 'A' }),
+      makePost({ title: 'B', published: '2026-04-02T00:00:00Z' }),
+      makePost({ title: 'C', published: '2026-04-03T00:00:00Z' }),
+    ];
+    const { kept, dropped } = partitionByKeepBound(posts, 1, () => false);
+    // Only the newest dated post is kept; the undated post sorts to the bottom.
+    expect(kept.map((post) => post.title)).toEqual(['C']);
+    expect(dropped.map((post) => post.title).sort()).toEqual(['A', 'B']);
+  });
+
+  test('keep bound of 0 drops all non-starred posts', () => {
+    const posts = [
+      makePost({ title: 'A', published: '2026-04-01T00:00:00Z' }),
+      makePost({ title: 'B-starred', published: '2026-04-02T00:00:00Z' }),
+    ];
+    const isStarred = (post: Subscription.Post) => post.title === 'B-starred';
+    const { kept, dropped } = partitionByKeepBound(posts, 0, isStarred);
+    expect(kept.map((post) => post.title)).toEqual(['B-starred']);
+    expect(dropped.map((post) => post.title)).toEqual(['A']);
+  });
+});

package/src/operations/util.ts

@@ -0,0 +1,65 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Database, Filter, Query } from '@dxos/echo';
+
+import { type Magazine, Subscription } from '../types';
+import { publishedTimestamp } from '../util/date';
+
+/**
+ * Resolves the Magazine's referenced feeds and the Posts in each feed's backing
+ * ECHO queue, filtering out Posts already curated into `magazine.posts` (and cross-feed duplicates).
+ */
+export const collectCandidates = (magazine: Magazine.Magazine) =>
+  Effect.gen(function* () {
+    const seen = new Set(magazine.posts.map((ref) => ref.target?.id ?? ref.uri));
+    const candidates: Array<{ post: Subscription.Post; feed: Subscription.Subscription }> = [];
+    for (const feedRef of magazine.feeds) {
+      const feed = yield* Database.load(feedRef);
+      const echoFeed = feed.feed?.target;
+      if (!echoFeed) {
+        continue;
+      }
+      const posts = yield* Database.query(Query.select(Filter.type(Subscription.Post)).from(echoFeed)).run;
+      for (const post of posts) {
+        if (seen.has(post.id)) {
+          continue;
+        }
+        seen.add(post.id);
+        candidates.push({ post, feed });
+      }
+    }
+    return candidates;
+  });
+
+/**
+ * Partitions posts into those to keep and those to drop, given a maximum
+ * non-starred retention bound. Starred posts are always kept; the remaining
+ * non-starred posts are sorted newest-first by `published` and the top
+ * `keep` retained.
+ *
+ * `isStarred` is injectable; callers with a starred-tag uri pass a predicate backed by it
+ * (defaults to treating nothing as starred).
+ */
+export const partitionByKeepBound = <T extends Subscription.Post>(
+  posts: readonly T[],
+  keep: number,
+  isStarred: (post: T) => boolean = () => false,
+): { kept: T[]; dropped: T[] } => {
+  const kept: T[] = [];
+  const candidates: T[] = [];
+  for (const post of posts) {
+    if (isStarred(post)) {
+      kept.push(post);
+    } else {
+      candidates.push(post);
+    }
+  }
+  candidates.sort((a, b) => publishedTimestamp(b.published) - publishedTimestamp(a.published));
+  const retained = candidates.slice(0, Math.max(0, keep));
+  const dropped = candidates.slice(Math.max(0, keep));
+  return { kept: [...kept, ...retained], dropped };
+};

package/src/paths.ts

@@ -0,0 +1,14 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { Paths } from '@dxos/app-toolkit';
+
+import { Magazine } from '#types';
+
+const { getSectionPath: getMagazinesPath, getObjectPath: getMagazinePath } = Paths.createTypeSectionPaths(
+  Magazine.Magazine,
+  { groupId: Paths.GroupSegments.content },
+);
+
+export { getMagazinePath, getMagazinesPath };

package/src/plugin.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Plugin } from '@dxos/app-framework';
+
+import { meta } from './meta';
+
+export const MagazinePlugin = Plugin.lazy(meta, () => import('#plugin'));
+
+export { MagazineOperationHandlerSet } from './operations';

package/src/plugin.workerd.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+export { MagazinePlugin, default } from './MagazinePlugin.workerd';
+export { MagazineOperationHandlerSet } from './operations';

package/src/skills/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+export { default as MagazineSkill } from './magazine-skill';

package/src/skills/magazine-skill.ts

@@ -0,0 +1,45 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { Skill, Template } from '@dxos/compute';
+import { trim } from '@dxos/util';
+
+import { FeedOperation } from '#types';
+import { Magazine } from '#types';
+
+const operations = [FeedOperation.FetchArticleContent];
+
+const make = () =>
+  Skill.make({
+    key: Magazine.SKILL_KEY,
+    name: 'Magazine Curator',
+    tools: Skill.toolDefinitions({ operations }),
+    instructions: Template.make({
+      // HOW only: input shape, tool usage, and the completeJob output contract. The editorial
+      // methodology (what to select, dedup, snippet/image guidance) and the magazine's topic live on
+      // the Routine's instructions (see Magazine.DEFAULT_INSTRUCTIONS), not here.
+      source: trim`
+        The candidate Posts are provided in the <input> as a JSON array under "candidates"; each
+        candidate has an "id" plus title, description, author, feedName, published, and link. The
+        editorial brief (the Topic and selection guidance) is provided in your instructions.
+
+        When a candidate's title and description are not enough to judge relevance, you MAY call
+        fetchArticleContent with the candidate's id to read the full article text (and find an image)
+        before deciding.
+
+        When you are done, call completeJob with the structured output:
+        { "posts": [{ "id": "<candidate id>", "snippet": "...", "imageUrl": "..." }, ...] }
+        listing the selected candidates in the order you want them shown. Only use ids that appear
+        in the input — never invent ids. Omit "imageUrl" when no image is available. Do not attempt to
+        add Posts yourself; the Magazine is updated mechanically from your output.
+      `,
+    }),
+  });
+
+const skill: Skill.Definition = {
+  key: Magazine.SKILL_KEY,
+  make,
+};
+
+export default skill;