dsh-ab-wechat-scrape 0.4.0

package/lib/index.d.ts

@@ -0,0 +1,219 @@
+/**
+ * Model-facing wechat_scrape tool. One call retrieves articles from WeChat
+ * official accounts and saves each as its own Markdown file named after the
+ * article title. A link may be an article page, or a list or collection page
+ * whose article links are parsed out and then scraped one by one the same way;
+ * a page that carries no article body is read as a list page. The HTTP layer
+ * paces, retries, carries the session cookies WeChat sets, and reports WeChat's
+ * block pages; a page whose body is built by client script, and a page the plain
+ * path is refused for, fall back to the optional browser renderer. Only a page
+ * that is still an article with a usable body after promotion filtering is ever
+ * written to a file: a block page, a withdrawn article, and an article the
+ * filtering left empty are reported as failures instead. The tool reads no
+ * session state beyond the workspace directory it writes into and returns the
+ * canonical value alone, so model-facing text and any replay read the same
+ * recorded call.
+ * @module @deepseek-ai/dsh-ab-wechat-scrape
+ */
+import type { Context } from '@deepseek-ai/cordis';
+import type { SandboxExecutionPolicy } from '@deepseek-ai/dsh-sandbox';
+import z from '@deepseek-ai/schemastery';
+import type FileSystem from '@deepseek-ai/dsh-fs';
+import { type CleanPolicy } from './clean.ts';
+import { type Fetcher } from './fetch.ts';
+import { type Renderer } from './render.ts';
+import type { DraftedArticle, WechatArgs, WechatFormat, WechatScrape } from './types.ts';
+export type * from './types.ts';
+export { articleDocument, renderArticle } from './document.ts';
+export { recordedArticle } from './store.ts';
+export { albumPageUrl, parseListPage } from './list.ts';
+export type { ListCursor, ListItem, ListPage } from './list.ts';
+export declare const name = "wechat-scrape";
+export declare const inject: string[];
+/** Wire name this plugin registers. */
+export declare const TOOL_NAME = "wechat_scrape";
+/** Body formats the tool accepts, in the order the model sees them. */
+export declare const BODY_FORMATS: readonly WechatFormat[];
+/**
+ * Name and order of the routing section this plugin contributes. Repository-owned
+ * tool sections allocate their order centrally; an external contribution states
+ * its own finite order and sits after the built-in tool band (TOOL_REPORT, 2900).
+ */
+export declare const SECTION_NAME = "tool:wechat_scrape";
+/** Order of {@link SECTION_NAME}, after the built-in tool sections. */
+export declare const SECTION_ORDER = 2970;
+/** Deployment-varying values for retrieval, list walking, filtering, output, pacing, and rendering. */
+export interface Config {
+    /** User agent every request and browser context sends. */
+    userAgent: string;
+    /** Referer every request and browser sends; empty omits the header. */
+    referer: string;
+    /** Cookie header for a logged-in session; empty sends none. */
+    cookie: string;
+    /** Whether cookies WeChat sets during a call are replayed for the rest of it. */
+    cookieJar: boolean;
+    /** Minimum milliseconds between two requests from this plugin instance. */
+    minIntervalMs: number;
+    /** Random milliseconds added to each wait, so the cadence is not a fixed period. */
+    minIntervalJitterMs: number;
+    /** Milliseconds the instance stands down for after WeChat refuses a page. */
+    blockCooldownMs: number;
+    /** Per-request timeout in milliseconds. */
+    timeoutMs: number;
+    /** Retries after the first attempt. */
+    maxRetries: number;
+    /** Base delay of the exponential backoff, in milliseconds. */
+    backoffBaseMs: number;
+    /** Longest wait a server-supplied retry hint may impose, in milliseconds. */
+    maxRetryAfterMs: number;
+    /** Maximum characters of the body the model reads; the saved file always holds all of it. */
+    maxChars: number;
+    /** Maximum image URLs the recorded value may carry. */
+    maxImages: number;
+    /** Maximum articles one call may scrape. */
+    maxArticles: number;
+    /** Collection pages one list walk may read; 0 reads until the collection ends. */
+    maxListPages: number;
+    /** Whether WeChat's advertising and follow-promotion blocks are removed from the body. */
+    filterAds: boolean;
+    /** Longest block the promotion text rule may drop, in characters. */
+    promoMaxChars: number;
+    /** Extra promotion rules, as regular-expression sources, beside the built-in ones. */
+    promoPatterns: string[];
+    /** Shortest filtered body, in characters, an article may carry and still be saved. */
+    minBodyChars: number;
+    /** Directory article files are written to; a relative value resolves against the session workspace. */
+    outputDir: string;
+    /** Longest article file name, in code points, before the extension. */
+    fileNameMaxChars: number;
+    /** auto renders a bodyless or refused page; always renders every page; never renders. */
+    renderer: 'auto' | 'never' | 'always';
+    /** Which renderer driver produces client-rendered HTML. */
+    rendererDriver: 'playwright' | 'http';
+    /** Endpoint the http renderer driver posts to. */
+    rendererUrl: string;
+    /** Playwright browser channel used when no executable path is set. */
+    browserChannel: string;
+    /** Explicit browser executable, overriding the channel. */
+    browserExecutablePath: string;
+    /** Whether the browser runs headless. */
+    headless: boolean;
+    /** Milliseconds an idle browser is kept before it is closed. */
+    browserIdleMs: number;
+    /** Selector the rendered article body must fill; empty skips the readiness wait. */
+    renderReadySelector: string;
+    /** Longest milliseconds a render waits for that body before reading the page as it stands. */
+    renderWaitMs: number;
+    /** Extra milliseconds to wait after navigation for lazy content. */
+    renderSettleMs: number;
+}
+/**
+ * Schemastery configuration for the tool. Defaults cover a single-machine
+ * deployment that writes into the calling session's workspace; the
+ * request-identity, list-depth, and output values are the ones a restricted or
+ * authenticated deployment overrides from cordis.yml.
+ */
+export declare const Config: z<Config>;
+/** Model-facing routing rule: when to make a call, which a schema cannot carry. */
+export declare const SECTION_TEXT: string;
+/** How one call reaches the network, the optional browser, and the file system. */
+export interface CollectDeps {
+    /** Fetcher that applies the deployment's headers, pacing, and retries. */
+    fetcher: Fetcher;
+    /** Renderer used for a page the plain path cannot serve; absent when disabled. */
+    renderer: Renderer | undefined;
+    /** The deployment values that bound and shape the result. */
+    config: Config;
+}
+/** One scrape call's dependencies, including where and how its files are written. */
+export interface ScrapeDeps extends CollectDeps {
+    /** Filesystem capability every article file is written through. */
+    fs: FileSystem;
+    /** Absolute directory every article file is written to. */
+    outputDir: string;
+    /** Per-call policy a confining filesystem fences the write by; absent on a bare backend. */
+    sandboxPolicy?: SandboxExecutionPolicy | undefined;
+}
+/**
+ * Validate the model-supplied link before any request is made.
+ * @param raw - the link as the model supplied it.
+ * @returns the trimmed link.
+ * @throws Error when the link is empty or names another host.
+ */
+export declare function normalizeUrl(raw: string): string;
+/**
+ * Collect the links one call asked for.
+ * @param args - the model-supplied call arguments.
+ * @param maxArticles - most links one call may start from.
+ * @returns the distinct links, in request order.
+ * @throws Error when the call names no link, a foreign link, or more links than the ceiling.
+ */
+export declare function requestUrls(args: WechatArgs, maxArticles: number): string[];
+/**
+ * Resolve the deployment's output directory.
+ *
+ * A relative value resolves against the workspace the calling session runs in.
+ * Under a confining filesystem that workspace is the per-call sandbox policy's
+ * root, not merely the session header's cwd: the policy is what the backend
+ * fences the write by, so a relative outputDir resolved against anything else
+ * would land outside the fence and be refused.
+ * @param configured - the row's outputDir value.
+ * @param cwd - the workspace this call resolves against, absent for a call without a session.
+ * @returns the absolute directory article files are written to.
+ */
+export declare function resolveOutputDir(configured: string, cwd: string | undefined): string;
+/**
+ * Cut text at the deployment ceiling, reporting whether anything was dropped.
+ * @param text - the whole converted body.
+ * @param maxChars - the deployment's excerpt ceiling.
+ * @returns the head within the ceiling and whether anything was dropped.
+ */
+export declare function clip(text: string, maxChars: number): {
+    text: string;
+    truncated: boolean;
+};
+/**
+ * The promotion rules one instance filters with.
+ * @param config - the deployment's filtering values.
+ * @returns the length ceiling and the compiled extra rules.
+ * @throws Error when an extra rule is not a valid regular expression.
+ */
+export declare function cleanPolicy(config: Config): CleanPolicy;
+/**
+ * Retrieve and parse one article.
+ * @param args - the model-supplied call arguments.
+ * @param deps - the instance's fetcher, optional renderer, and bounds.
+ * @param signal - caller cancellation.
+ * @returns the drafted article, before it is saved.
+ * @throws Error for an invalid link, a block page, a failed request, or a link that is a list page.
+ */
+export declare function collectArticle(args: WechatArgs, deps: CollectDeps, signal?: AbortSignal): Promise<DraftedArticle>;
+/**
+ * Scrape every link one call named, saving each article to its own file. A link
+ * may be an article or a list page; a list page's article links are scraped the
+ * same way as a named article. One failing link or article does not discard
+ * what the other links produced.
+ * @param args - the model-supplied call arguments.
+ * @param deps - the instance's fetcher, renderer, bounds, and output directory.
+ * @param signal - caller cancellation.
+ * @returns the saved articles, the list pages walked, and the links that failed.
+ * @throws Error when the call names no usable link, or when the caller cancels.
+ */
+export declare function scrape(args: WechatArgs, deps: ScrapeDeps, signal?: AbortSignal): Promise<WechatScrape>;
+/**
+ * Compose the model-facing text of one call's result.
+ * @param value - the canonical scrape value.
+ * @returns every list page walked, every saved article, then every failure.
+ */
+export declare function renderScrape(value: WechatScrape): string;
+/**
+ * Contribute the routing section, own the renderer's lifetime, and register the
+ * wechat_scrape tool. The section is empty wherever the tool is not visible in
+ * that scope, so a restricted composition is not told to call a hidden tool.
+ * @param ctx - registrant context carrying the tool, system-prompt, and filesystem registries.
+ * @param config - the deployment's retrieval, list, filtering, output, and renderer values.
+ * @throws Error when a confining filesystem is mounted with no sandbox-policy service,
+ * because no per-call workspace root could then be resolved for the article writes.
+ */
+export declare function apply(ctx: Context, config: Config): void;
+//# sourceMappingURL=index.d.ts.map

package/lib/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAClD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAA;AACtE,OAAO,CAAC,MAAM,0BAA0B,CAAA;AAIxC,OAAO,KAAK,UAAU,MAAM,qBAAqB,CAAA;AAEjD,OAAO,EAAsC,KAAK,WAAW,EAAE,MAAM,YAAY,CAAA;AAEjF,OAAO,EAAiB,KAAK,OAAO,EAAE,MAAM,YAAY,CAAA;AAIxD,OAAO,EAAkB,KAAK,QAAQ,EAAE,MAAM,aAAa,CAAA;AAE3D,OAAO,KAAK,EACV,cAAc,EAEd,UAAU,EAGV,YAAY,EAEZ,YAAY,EACb,MAAM,YAAY,CAAA;AAEnB,mBAAmB,YAAY,CAAA;AAC/B,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAC5C,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,WAAW,CAAA;AACvD,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAE/D,eAAO,MAAM,IAAI,kBAAkB,CAAA;AACnC,eAAO,MAAM,MAAM,UAAkC,CAAA;AAErD,uCAAuC;AACvC,eAAO,MAAM,SAAS,kBAAkB,CAAA;AAExC,uEAAuE;AACvE,eAAO,MAAM,YAAY,EAAE,SAAS,YAAY,EAAyB,CAAA;AAEzE;;;;GAIG;AACH,eAAO,MAAM,YAAY,uBAAuB,CAAA;AAEhD,uEAAuE;AACvE,eAAO,MAAM,aAAa,OAAO,CAAA;AAKjC,uGAAuG;AACvG,MAAM,WAAW,MAAM;IACrB,0DAA0D;IAC1D,SAAS,EAAE,MAAM,CAAA;IACjB,uEAAuE;IACvE,OAAO,EAAE,MAAM,CAAA;IACf,+DAA+D;IAC/D,MAAM,EAAE,MAAM,CAAA;IACd,iFAAiF;IACjF,SAAS,EAAE,OAAO,CAAA;IAClB,2EAA2E;IAC3E,aAAa,EAAE,MAAM,CAAA;IACrB,oFAAoF;IACpF,mBAAmB,EAAE,MAAM,CAAA;IAC3B,6EAA6E;IAC7E,eAAe,EAAE,MAAM,CAAA;IACvB,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAA;IACjB,uCAAuC;IACvC,UAAU,EAAE,MAAM,CAAA;IAClB,8DAA8D;IAC9D,aAAa,EAAE,MAAM,CAAA;IACrB,6EAA6E;IAC7E,eAAe,EAAE,MAAM,CAAA;IACvB,6FAA6F;IAC7F,QAAQ,EAAE,MAAM,CAAA;IAChB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAA;IACjB,4CAA4C;IAC5C,WAAW,EAAE,MAAM,CAAA;IACnB,kFAAkF;IAClF,YAAY,EAAE,MAAM,CAAA;IACpB,0FAA0F;IAC1F,SAAS,EAAE,OAAO,CAAA;IAClB,qEAAqE;IACrE,aAAa,EAAE,MAAM,CAAA;IACrB,sFAAsF;IACtF,aAAa,EAAE,MAAM,EAAE,CAAA;IACvB,sFAAsF;IACtF,YAAY,EAAE,MAAM,CAAA;IACpB,uGAAuG;IACvG,SAAS,EAAE,MAAM,CAAA;IACjB,uEAAuE;IACvE,gBAAgB,EAAE,MAAM,CAAA;IACxB,yFAAyF;IACzF,QAAQ,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAA;IACrC,2DAA2D;IAC3D,cAAc,EAAE,YAAY,GAAG,MAAM,CAAA;IACrC,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAA;IACnB,sEAAsE;IACtE,cAAc,EAAE,MAAM,CAAA;IACtB,2DAA2D;IAC3D,qBAAqB,EAAE,MAAM,CAAA;IAC7B,yCAAyC;IACzC,QAAQ,EAAE,OAAO,CAAA;IACjB,gEAAgE;IAChE,aAAa,EAAE,MAAM,CAAA;IACrB,oFAAoF;IACpF,mBAAmB,EAAE,MAAM,CAAA;IAC3B,8FAA8F;IAC9F,YAAY,EAAE,MAAM,CAAA;IACpB,oEAAoE;IACpE,cAAc,EAAE,MAAM,CAAA;CACvB;AAED;;;;;GAKG;AACH,eAAO,MAAM,MAAM,EAAE,CAAC,CAAC,MAAM,CAgC3B,CAAA;AASF,mFAAmF;AACnF,eAAO,MAAM,YAAY,QAId,CAAA;AAEX,mFAAmF;AACnF,MAAM,WAAW,WAAW;IAC1B,0EAA0E;IAC1E,OAAO,EAAE,OAAO,CAAA;IAChB,kFAAkF;IAClF,QAAQ,EAAE,QAAQ,GAAG,SAAS,CAAA;IAC9B,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAA;CACf;AAED,qFAAqF;AACrF,MAAM,WAAW,UAAW,SAAQ,WAAW;IAC7C,mEAAmE;IACnE,EAAE,EAAE,UAAU,CAAA;IACd,2DAA2D;IAC3D,SAAS,EAAE,MAAM,CAAA;IACjB,4FAA4F;IAC5F,aAAa,CAAC,EAAE,sBAAsB,GAAG,SAAS,CAAA;CACnD;AAoDD;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAKhD;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,EAAE,CAY3E;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAEpF;AAED;;;;;GAKG;AACH,wBAAgB,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,OAAO,CAAA;CAAE,CAGzF;AAiDD;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAevD;AAsKD;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAClC,IAAI,EAAE,UAAU,EAChB,IAAI,EAAE,WAAW,EACjB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,cAAc,CAAC,CAOzB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAkE5G;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAexD;AAED;;;;;;;;GAQG;AACH,wBAAgB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAoKxD"}