@dxos/plugin-magazine 0.0.0

package/src/operations/curate-magazine.skill.test.ts

@@ -0,0 +1,167 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+// Run in the node environment (not the package-default happy-dom): the agent's outbound Anthropic
+// call goes through real `fetch`; happy-dom's fetch enforces CORS and blocks the cross-origin request.
+// extractArticle falls back to defuddle/node (linkedom) when DOMParser is absent, so the real
+// fetchArticleContent tool works here without a DOM polyfill.
+// @vitest-environment node
+
+import { describe, it } from '@effect/vitest';
+import * as Effect from 'effect/Effect';
+import { afterEach, beforeEach, vi } from 'vitest';
+
+import { AgentHandlers } from '@dxos/assistant-toolkit';
+import { Operation } from '@dxos/compute';
+import { Database, Feed, Obj, Ref, Tag } from '@dxos/echo';
+import { TestHelpers } from '@dxos/effect/testing';
+import { AssistantTestLayer } from '@dxos/functions-runtime/testing';
+import { EntityId } from '@dxos/keys';
+import { StateMap, TagIndex, Text } from '@dxos/schema';
+
+import { MagazineSkill } from '../skills';
+import { FeedOperation, Magazine, Subscription } from '../types';
+import { MagazineOperationHandlerSet } from './index';
+
+EntityId.dangerouslyDisableRandomness();
+
+const TestLayer = AssistantTestLayer({
+  // MagazineOperationHandlerSet provides CurateMagazine + SyncFeed + FetchArticleContent (the skill
+  // tool); AgentHandlers provides the RunInstructions handler the curate operation invokes.
+  operationHandlers: [MagazineOperationHandlerSet, AgentHandlers],
+  types: [
+    Feed.Feed,
+    Subscription.Subscription,
+    Subscription.Post,
+    Magazine.Magazine,
+    Tag.Tag,
+    Text.Text,
+    StateMap.StateMap,
+    TagIndex.TagIndex,
+  ],
+  skills: [MagazineSkill.make()],
+});
+
+/**
+ * Candidate posts seeded into the subscription's backing queue: two clearly on-topic for a
+ * space/astronomy magazine, two clearly off-topic.
+ */
+const CANDIDATES = [
+  {
+    guid: 'artemis-ii',
+    title: "NASA's Artemis II crew completes lunar flyby rehearsal",
+    description:
+      'The four astronauts ran a full simulation of the crewed Moon flyby, practicing re-entry and recovery procedures ahead of next year’s launch.',
+    onTopic: true,
+  },
+  {
+    guid: 'jwst-galaxy',
+    title: 'James Webb telescope confirms the most distant galaxy yet seen',
+    description:
+      'Astronomers used JWST spectroscopy to confirm a galaxy as it appeared 290 million years after the Big Bang, a new record for cosmic distance.',
+    onTopic: true,
+  },
+  {
+    guid: 'retail-earnings',
+    title: 'Retail chain beats quarterly earnings estimates',
+    description:
+      'Shares rose 6% after the company reported stronger-than-expected holiday sales and raised its full-year profit guidance.',
+    onTopic: false,
+  },
+  {
+    guid: 'js-bundler',
+    title: 'New JavaScript build tool promises 10x faster bundling',
+    description:
+      'The Rust-based bundler aims to replace existing toolchains for large web applications with incremental compilation.',
+    onTopic: false,
+  },
+];
+
+/** Minimal article page so the real extractArticle (defuddle) has a body to parse. */
+const articleHtml = (title: string, body: string) =>
+  `<!doctype html><html><head><title>${title}</title></head><body><article><h1>${title}</h1><p>${body}</p></article></body></html>`;
+
+// Preserve the real fetch so the stub can pass the agent's Anthropic call through (replay is
+// memoized, so no real call is made then; generation uses the live API key).
+// eslint-disable-next-line no-restricted-globals
+const realFetch = globalThis.fetch.bind(globalThis);
+
+describe('CurateMagazine (LLM)', () => {
+  // Stub only the network boundary: serve canned HTML for the candidate article URLs (so the real
+  // fetchArticleContent tool runs deterministically), and pass everything else (Anthropic) through.
+  beforeEach(() => {
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
+        const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
+        const candidate = CANDIDATES.find((entry) => url.includes(entry.guid));
+        if (candidate) {
+          return new Response(articleHtml(candidate.title, candidate.description), {
+            status: 200,
+            headers: { 'content-type': 'text/html' },
+          });
+        }
+        return realFetch(input, init);
+      }),
+    );
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+  });
+
+  it.effect(
+    'selects only the posts matching the magazine topic',
+    Effect.fnUntraced(
+      function* ({ expect }) {
+        // Subscription with no `url` → the SyncFeed phase is skipped (no network); curation reads the
+        // posts we seed directly into the backing queue below.
+        const subscription = yield* Database.add(
+          Subscription.makeSubscription({ name: 'Tech & Science Daily', type: 'rss' }),
+        );
+        yield* Database.flush();
+
+        const postFeed = yield* Database.load(subscription.feed);
+        const posts = CANDIDATES.map((candidate) =>
+          Obj.make(Subscription.Post, {
+            title: candidate.title,
+            description: candidate.description,
+            link: `https://example.com/${candidate.guid}`,
+            guid: candidate.guid,
+            published: '2026-05-01T00:00:00Z',
+            source: Ref.make(subscription),
+          }),
+        );
+        yield* Feed.append(postFeed, posts);
+        yield* Database.flush();
+
+        const magazine = yield* Database.add(
+          Magazine.make({
+            name: 'The Cosmos',
+            feeds: [Ref.make(subscription)],
+            instructions:
+              'Curate articles about space exploration and astronomy — missions, spacecraft, telescopes, and astrophysics discoveries. Exclude unrelated business, finance, or software-tooling news.',
+          }),
+        );
+        yield* Database.flush();
+
+        const result = yield* Operation.invoke(FeedOperation.CurateMagazine, { magazine: Ref.make(magazine) });
+
+        // Load the curated post refs to read their titles (`ref.target` is not hydrated synchronously).
+        const curatedPosts = yield* Effect.promise(() => Promise.all(magazine.posts.map((ref) => ref.load())));
+        const curatedTitles = curatedPosts.map((post) => post.title);
+
+        // The two space/astronomy posts are curated; the business + tooling posts are not.
+        expect(curatedTitles).toContain(CANDIDATES[0].title);
+        expect(curatedTitles).toContain(CANDIDATES[1].title);
+        expect(curatedTitles).not.toContain(CANDIDATES[2].title);
+        expect(curatedTitles).not.toContain(CANDIDATES[3].title);
+        expect(result.curated).toBe(2);
+      },
+      Effect.provide(TestLayer),
+      TestHelpers.provideTestContext,
+    ),
+    { timeout: 240_000 },
+  );
+});

package/src/operations/curate-magazine.test.ts

@@ -0,0 +1,91 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { afterEach, beforeEach, describe, expect, test } from 'vitest';
+
+import { Feed, Obj, Ref, Tag } from '@dxos/echo';
+import { EchoTestBuilder } from '@dxos/echo-client/testing';
+import { StateMap, TagIndex, Text } from '@dxos/schema';
+
+import { Magazine, Subscription } from '../types';
+import { applyKeep, resolveSelected } from './curate-magazine';
+
+describe('applyKeep', () => {
+  let builder: EchoTestBuilder;
+
+  beforeEach(async () => {
+    builder = await new EchoTestBuilder().open();
+  });
+
+  afterEach(async () => {
+    await builder.close();
+  });
+
+  const setup = async () => {
+    const { db } = await builder.createDatabase({
+      types: [
+        Feed.Feed,
+        Subscription.Subscription,
+        Subscription.Post,
+        Magazine.Magazine,
+        Tag.Tag,
+        Text.Text,
+        StateMap.StateMap,
+        TagIndex.TagIndex,
+      ],
+    });
+    return { db };
+  };
+
+  const makePost = (published: string): Subscription.Post =>
+    Obj.make(Subscription.Post, { title: `Post ${published}`, description: 'body', published });
+
+  test('keeps the newest posts up to the bound', async () => {
+    const { db } = await setup();
+    const posts = ['2026-01-01', '2026-01-02', '2026-01-03'].map((date) =>
+      Ref.make(db.add(makePost(`${date}T00:00:00Z`))),
+    );
+
+    const kept = applyKeep(posts, 2, undefined);
+    const keptDates = kept.map((ref) => ref.target?.published);
+    expect(kept).toHaveLength(2);
+    expect(keptDates).toContain('2026-01-03T00:00:00Z');
+    expect(keptDates).toContain('2026-01-02T00:00:00Z');
+    expect(keptDates).not.toContain('2026-01-01T00:00:00Z');
+  });
+
+  test('no-ops when within the bound', async () => {
+    const { db } = await setup();
+    const posts = ['2026-01-01', '2026-01-02'].map((date) => Ref.make(db.add(makePost(`${date}T00:00:00Z`))));
+    expect(applyKeep(posts, 10, undefined)).toHaveLength(2);
+  });
+});
+
+describe('resolveSelected', () => {
+  let builder: EchoTestBuilder;
+
+  beforeEach(async () => {
+    builder = await new EchoTestBuilder().open();
+  });
+
+  afterEach(async () => {
+    await builder.close();
+  });
+
+  test('maps ids to posts in order, dropping unknown and duplicate ids', async () => {
+    const { db } = await builder.createDatabase({ types: [Subscription.Post] });
+    const posts = ['a', 'b', 'c'].map((title) => db.add(Obj.make(Subscription.Post, { title })));
+    await db.flush();
+    const candidates = posts.map((post) => ({ post }));
+
+    const selected = resolveSelected(candidates, [
+      { id: posts[2].id },
+      { id: 'missing-id' },
+      { id: posts[0].id },
+      { id: posts[2].id },
+    ]);
+
+    expect(selected.map(({ post }) => post.id)).toEqual([posts[2].id, posts[0].id]);
+  });
+});

package/src/operations/curate-magazine.ts

@@ -0,0 +1,178 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Option from 'effect/Option';
+import * as Schema from 'effect/Schema';
+
+import { RunInstructions } from '@dxos/assistant-toolkit';
+import { Operation } from '@dxos/compute';
+import { Database, Obj, Ref } from '@dxos/echo';
+import { type EntityId, type SpaceId } from '@dxos/keys';
+import { log } from '@dxos/log';
+
+import { FeedOperation, Magazine, Subscription } from '../types';
+import { collectCandidates, partitionByKeepBound } from './util';
+
+export default FeedOperation.CurateMagazine.pipe(
+  Operation.withHandler(
+    Effect.fnUntraced(function* ({ magazine: magazineRef }) {
+      const magazine = yield* Effect.promise(() => magazineRef.load());
+
+      const validFeeds = yield* loadValidFeeds(magazine);
+      const synced = yield* syncFeeds(validFeeds);
+
+      // Select matching Posts via the agent (single-shot structured output), then add them mechanically.
+      const candidates = yield* collectCandidates(magazine);
+      const spaceId = Obj.getDatabase(magazine)?.spaceId;
+      const selectedEntries =
+        candidates.length > 0 && spaceId ? yield* selectPostIds(magazine, candidates, spaceId) : [];
+      const selected = resolveSelected(candidates, selectedEntries);
+
+      // Build the next posts list as a pure function of (existing curated + newly selected), bounded
+      // by the magazine's `keep`, then commit it in one update. collectCandidates already excludes
+      // posts already curated, so the additions are simply appended (resolveSelected deduped them).
+      const db = Obj.getDatabase(magazine);
+      const starredUri = db ? yield* Effect.promise(() => Subscription.findSystemTagUri(db, 'starred')) : undefined;
+      const merged = [...magazine.posts, ...selected.map(({ post }) => Ref.make(post))];
+      const nextPosts = applyKeep(merged, magazine.keep ?? Subscription.DEFAULT_KEEP, starredUri);
+      const curated = selected.length;
+
+      const changed =
+        nextPosts.length !== magazine.posts.length ||
+        nextPosts.some((ref, index) => ref.uri !== magazine.posts[index]?.uri);
+      if (changed) {
+        Obj.update(magazine, (magazine) => {
+          magazine.posts = nextPosts;
+        });
+      }
+
+      // Write agent-generated snippet/imageUrl into per-post magazine state.
+      for (const { post, snippet, imageUrl } of selected) {
+        if (snippet || imageUrl) {
+          Magazine.patchPostState(magazine, post.id as EntityId, { snippet, imageUrl });
+        }
+      }
+
+      return { synced, curated };
+    }),
+  ),
+  Operation.opaqueHandler,
+);
+
+// -- Helpers --
+
+/** Bound on concurrent feed syncs. */
+const SYNC_CONCURRENCY = 8;
+
+/** Loads each referenced feed (and its backing ECHO feed), tolerating individual failures, keeping only syncable feeds. */
+const loadValidFeeds = (magazine: Magazine.Magazine) =>
+  Effect.forEach(magazine.feeds, (ref) =>
+    Effect.gen(function* () {
+      const feed = yield* Database.load(ref);
+      if (feed.feed) {
+        yield* Database.load(feed.feed); // Hydrate the backing ECHO feed.
+      }
+      return feed;
+    }).pipe(
+      Effect.tapError((error) => Effect.sync(() => log.catch(error))),
+      Effect.option,
+    ),
+  ).pipe(Effect.map((feeds) => feeds.flatMap(Option.toArray).filter((feed) => Boolean(feed.url))));
+
+/** Syncs all feeds in parallel, tolerating per-feed failures; resolves to the count synced successfully. */
+const syncFeeds = (validFeeds: readonly Subscription.Subscription[]) =>
+  Effect.forEach(
+    validFeeds,
+    (feed) =>
+      Operation.invoke(
+        FeedOperation.SyncFeed,
+        { feed: Ref.make(feed) },
+        { spaceId: Obj.getDatabase(feed)?.spaceId },
+      ).pipe(
+        Effect.as(true),
+        Effect.catchAll((error) => Effect.sync(() => (log.catch(error, { feedUrl: feed.url }), false))),
+      ),
+    { concurrency: SYNC_CONCURRENCY },
+  ).pipe(Effect.map((results) => results.filter(Boolean).length));
+
+/**
+ * Runs the curation agent over the candidate summaries and resolves to the selected Post entries.
+ * The magazine's persisted Instructions (created with the magazine) carries the editorial brief and
+ * references the Magazine skill, which RunInstructions resolves at run time. No instructions → no selection.
+ * Tolerates agent/parse failures (logs → no selection).
+ */
+const selectPostIds = (
+  magazine: Magazine.Magazine,
+  candidates: ReadonlyArray<{ post: Subscription.Post; feed: Subscription.Subscription }>,
+  spaceId: SpaceId,
+) =>
+  Effect.gen(function* () {
+    if (!magazine.instructions) {
+      return [] as readonly (typeof Magazine.CurationOutput.Type.posts)[number][];
+    }
+    const input = {
+      candidates: candidates.map(({ post, feed }) => ({
+        id: post.id,
+        feedName: feed.name,
+        title: post.title,
+        description: post.description,
+        author: post.author,
+        published: post.published,
+        link: post.link,
+      })),
+    };
+
+    return yield* Operation.invoke(RunInstructions, { instructions: magazine.instructions, input }, { spaceId }).pipe(
+      Effect.flatMap(Schema.decodeUnknown(Magazine.CurationOutput)),
+      Effect.map((output) => output.posts),
+      Effect.catchAll((error) =>
+        Effect.sync(() => {
+          log.warn('curation selection failed', { error });
+          return [] as readonly (typeof Magazine.CurationOutput.Type.posts)[number][];
+        }),
+      ),
+    );
+  });
+
+type SelectedEntry = {
+  post: Subscription.Post;
+  snippet: string | undefined;
+  imageUrl: string | undefined;
+};
+
+/** Resolves the agent's selected entries back to candidate Posts, preserving order and dropping unknown/duplicate ids. */
+export const resolveSelected = (
+  candidates: ReadonlyArray<{ post: Subscription.Post }>,
+  entries: readonly { id: string; snippet?: string; imageUrl?: string }[],
+): SelectedEntry[] => {
+  const byId = new Map(candidates.map(({ post }) => [post.id, post]));
+  const seen = new Set<string>();
+  const selected: SelectedEntry[] = [];
+  for (const { id, snippet, imageUrl } of entries) {
+    const post = byId.get(id);
+    if (post && !seen.has(id)) {
+      seen.add(id);
+      selected.push({ post, snippet, imageUrl });
+    }
+  }
+  return selected;
+};
+
+/**
+ * Bounds a curated posts ref list to `keep` total (non-starred) posts: keeps all starred posts and
+ * unresolved refs, plus the `keep` newest non-starred by published date. Pure; returns the retained
+ * refs. Delegates the sort/slice/starred partition to {@link partitionByKeepBound}.
+ */
+export const applyKeep = (
+  posts: readonly Ref.Ref<Subscription.Post>[],
+  keep: number,
+  starredUri: string | undefined,
+): Ref.Ref<Subscription.Post>[] => {
+  const isStarred = (post: Subscription.Post) => Subscription.hasTag(post.source?.target, post.id, starredUri);
+  const resolved = posts.map((ref) => ref.target).filter((post): post is Subscription.Post => post !== undefined);
+  const unresolved = posts.filter((ref) => !ref.target);
+  const { kept } = partitionByKeepBound(resolved, keep, isStarred);
+  return [...kept.map((post) => Ref.make(post)), ...unresolved];
+};

package/src/operations/extraction/article.test.ts

@@ -0,0 +1,154 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { describe, expect, test } from 'vitest';
+
+import { extractArticle, pruneTrailingChrome } from './article';
+
+const parse = (html: string): Document => new DOMParser().parseFromString(html, 'text/html');
+
+describe('extractArticle', () => {
+  test('returns empty result for empty input', async () => {
+    const result = await extractArticle('');
+    expect(result.markdown).toBe('');
+    expect(result.imageUrls).toEqual([]);
+  });
+
+  test('strips trailing tag-cloud (heading + /Tag/ link list)', async () => {
+    // theregister.com shape: a "Narrower topics" / "Broader topics" heading
+    // followed by a `<ul>` of links to /Tag/* routes inside the article.
+    const html = `
+      <html><head><title>Test</title></head>
+      <body>
+        <article>
+          <h1>The article</h1>
+          <p>Body paragraph one with substantive text so defuddle treats this as an article. Lorem ipsum dolor sit amet.</p>
+          <p>Body paragraph two. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium.</p>
+          <p>Closing line.</p>
+          <h3>Narrower topics</h3>
+          <ul>
+            <li><a href="/Tag/AIOps/">AIOps</a></li>
+            <li><a href="/Tag/Anthropic/">Anthropic</a></li>
+            <li><a href="/Tag/ChatGPT/">ChatGPT</a></li>
+          </ul>
+          <h3>Broader topics</h3>
+          <ul>
+            <li><a href="/Tag/Self-driving/">Self-driving</a></li>
+          </ul>
+        </article>
+      </body></html>
+    `;
+    const result = await extractArticle(html, 'https://example.com/test');
+    expect(result.markdown).not.toMatch(/Narrower topics/i);
+    expect(result.markdown).not.toMatch(/Broader topics/i);
+    expect(result.markdown).not.toMatch(/AIOps/);
+    expect(result.markdown).not.toMatch(/Self-driving/);
+    expect(result.markdown).toMatch(/Body paragraph one/);
+    expect(result.markdown).toMatch(/Closing line/);
+  });
+
+  test('strips trailing block by class hint even without a giveaway heading', async () => {
+    // No "Narrower topics"-style label — the trailing block is recognised
+    // by its `class="related-articles"` hint alone. The text-pattern trim
+    // could not catch this; the DOM rules can.
+    const html = `
+      <html><body>
+        <article>
+          <h1>Essay</h1>
+          <p>Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur.</p>
+          <p>Final paragraph of the body. Vel illum qui dolorem eum fugiat quo voluptas nulla pariatur.</p>
+          <div class="related-articles">
+            <a href="/articles/abc">Another piece</a>
+            <a href="/articles/def">Yet another</a>
+            <a href="/articles/ghi">A third</a>
+          </div>
+        </article>
+      </body></html>
+    `;
+    const result = await extractArticle(html, 'https://example.com/essay');
+    expect(result.markdown).not.toMatch(/Another piece/);
+    expect(result.markdown).not.toMatch(/Yet another/);
+    expect(result.markdown).toMatch(/Final paragraph of the body/);
+  });
+
+  test('strips trailing <aside> nested inside the article', async () => {
+    const html = `
+      <html><body>
+        <main>
+          <h1>News piece</h1>
+          <p>News body. At vero eos et accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum.</p>
+          <p>More body. Nam libero tempore, cum soluta nobis est eligendi optio cumque nihil impedit quo minus.</p>
+          <aside class="more-from-author">
+            <h3>More from this author</h3>
+            <ul><li><a href="/by/jane/1">One</a></li><li><a href="/by/jane/2">Two</a></li></ul>
+          </aside>
+        </main>
+      </body></html>
+    `;
+    const result = await extractArticle(html, 'https://example.com/news');
+    expect(result.markdown).not.toMatch(/More from this author/i);
+    expect(result.markdown).toMatch(/News body/);
+  });
+
+  test('pruneTrailingChrome preserves a heading that is naturally last (no chrome removed before it)', () => {
+    // Direct DOM test — defuddle's later passes may independently drop a
+    // lone trailing heading, so we assert against the post-prune DOM rather
+    // than the final markdown to isolate the pruner's guard.
+    const doc = parse(`
+      <html><body>
+        <article>
+          <h1>The piece</h1>
+          <p>Body paragraph.</p>
+          <h2>Conclusion</h2>
+        </article>
+      </body></html>
+    `);
+    pruneTrailingChrome(doc);
+    const article = doc.querySelector('article')!;
+    expect(article.querySelector('h2')?.textContent).toBe('Conclusion');
+    expect(article.querySelectorAll('h2,p').length).toBe(2);
+  });
+
+  test('pruneTrailingChrome strips the heading paired with a chrome block it titles', () => {
+    const doc = parse(`
+      <html><body>
+        <article>
+          <h1>The piece</h1>
+          <p>Body paragraph.</p>
+          <h2>Narrower topics</h2>
+          <ul>
+            <li><a href="/Tag/AIOps/">AIOps</a></li>
+            <li><a href="/Tag/Anthropic/">Anthropic</a></li>
+          </ul>
+        </article>
+      </body></html>
+    `);
+    pruneTrailingChrome(doc);
+    const article = doc.querySelector('article')!;
+    expect(article.querySelector('ul')).toBeNull();
+    expect(article.querySelector('h2')).toBeNull();
+    expect(article.querySelectorAll('p').length).toBe(1);
+  });
+
+  test('does not strip a "Topics" section in the middle of the article', async () => {
+    // A section titled "Topics" appears mid-article followed by another body
+    // section. Trailing-chrome pruning must leave it alone.
+    const html = `
+      <html><body>
+        <article>
+          <h1>An essay</h1>
+          <p>Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.</p>
+          <h2>Topics</h2>
+          <p>Here are the topics this essay covers in prose form. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip.</p>
+          <h2>Conclusion</h2>
+          <p>That wraps it up. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.</p>
+        </article>
+      </body></html>
+    `;
+    const result = await extractArticle(html, 'https://example.com/essay');
+    expect(result.markdown).toMatch(/Topics/);
+    expect(result.markdown).toMatch(/Conclusion/);
+    expect(result.markdown).toMatch(/That wraps it up/);
+  });
+});