@tokensapi/dsh-progressive-tools 0.1.0

package/lib/catalog.js

@@ -0,0 +1,267 @@
+const WORD_PATTERN = /[\p{L}\p{N}]+/gu;
+function normalize(value) {
+    return value
+        .replace(/([a-z\d])([A-Z])/g, '$1 $2')
+        .replaceAll('_', ' ')
+        .replaceAll('-', ' ')
+        .toLocaleLowerCase('en-US')
+        .match(WORD_PATTERN)
+        ?.join(' ') ?? '';
+}
+const CJK_RUN_PATTERN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu;
+/**
+ * CJK text has no space-delimited word boundaries, so whole-phrase tokens
+ * would never overlap between queries and definitions. Character bigrams give
+ * both sides comparable terms without a segmentation dictionary.
+ */
+function tokens(value) {
+    const normalized = normalize(value);
+    if (normalized === '')
+        return [];
+    const result = [];
+    for (const token of normalized.split(' ')) {
+        result.push(token);
+        for (const run of token.match(CJK_RUN_PATTERN) ?? []) {
+            for (let index = 0; index + 1 < run.length; index += 1) {
+                const bigram = run.slice(index, index + 2);
+                if (bigram !== token)
+                    result.push(bigram);
+            }
+        }
+    }
+    return result;
+}
+function wildcard(pattern) {
+    const escaped = pattern.replace(/[|\\{}()[\]^$+?.]/g, String.raw `\$&`);
+    return new RegExp(`^${escaped.replaceAll('*', '.*')}$`, 'i');
+}
+export function matchesToolName(name, patterns) {
+    return patterns.some(pattern => wildcard(pattern).test(name));
+}
+function parameterKeys(value) {
+    if (value === null || typeof value !== 'object' || Array.isArray(value))
+        return [];
+    const record = value;
+    const own = Object.keys(record);
+    return [...own, ...Object.values(record).flatMap(parameterKeys)];
+}
+function schemaSearchText(value) {
+    if (value === null || typeof value !== 'object')
+        return typeof value === 'string' ? value : '';
+    if (Array.isArray(value))
+        return value.map(schemaSearchText).join(' ');
+    return Object.entries(value)
+        .flatMap(([key, child]) => [key, schemaSearchText(child)])
+        .join(' ');
+}
+export function estimateSchemaTokens(schema, charactersPerToken) {
+    return Math.max(1, Math.ceil(JSON.stringify(schema).length / charactersPerToken));
+}
+function createCatalogTool(schema, charactersPerToken) {
+    return {
+        ...schema,
+        estimatedTokens: estimateSchemaTokens(schema, charactersPerToken),
+        searchText: normalize(`${schema.name} ${schema.description} ${parameterKeys(schema.parameters).join(' ')} ${schemaSearchText(schema.parameters)}`),
+    };
+}
+function createGroup(id, description, aliases, tools) {
+    return {
+        id,
+        description,
+        aliases,
+        tools,
+        estimatedTokens: tools.reduce((total, tool) => total + tool.estimatedTokens, 0),
+        searchText: normalize(`${id} ${description} ${aliases.join(' ')} ${tools.map(tool => tool.searchText).join(' ')}`),
+    };
+}
+/**
+ * Prefixes shared by unrelated tools across plugins. Merging on them would
+ * bundle strangers (get_goal with get_image_generation_task), so these tools
+ * stay single-tool groups instead.
+ */
+const GENERIC_AUTO_PREFIXES = new Set([
+    'get', 'set', 'list', 'create', 'delete', 'update', 'add', 'remove',
+    'cancel', 'run', 'start', 'stop', 'send', 'read', 'write', 'new', 'check',
+]);
+function automaticGroupId(name, prefixCounts) {
+    const prefix = name.includes('_') ? name.slice(0, name.indexOf('_')) : name;
+    if (GENERIC_AUTO_PREFIXES.has(prefix))
+        return name;
+    return (prefixCounts.get(prefix) ?? 0) >= 2 ? prefix : name;
+}
+export function buildCatalog(schemas, configuredGroups, charactersPerToken, excludedNames = new Set()) {
+    const tools = new Map();
+    for (const schema of schemas) {
+        if (!excludedNames.has(schema.name))
+            tools.set(schema.name, createCatalogTool(schema, charactersPerToken));
+    }
+    const assigned = new Set();
+    const groups = new Map();
+    for (const config of configuredGroups) {
+        const members = [...tools.values()].filter(tool => !assigned.has(tool.name)
+            && matchesToolName(tool.name, config.include)
+            && !matchesToolName(tool.name, config.exclude ?? []));
+        if (members.length === 0)
+            continue;
+        for (const tool of members)
+            assigned.add(tool.name);
+        groups.set(config.id, createGroup(config.id, config.description ?? `${config.id} tools`, config.aliases ?? [], members));
+    }
+    const unassigned = [...tools.values()].filter(tool => !assigned.has(tool.name));
+    const prefixCounts = new Map();
+    for (const tool of unassigned) {
+        const prefix = tool.name.includes('_') ? tool.name.slice(0, tool.name.indexOf('_')) : tool.name;
+        prefixCounts.set(prefix, (prefixCounts.get(prefix) ?? 0) + 1);
+    }
+    const automatic = new Map();
+    for (const tool of unassigned) {
+        const id = automaticGroupId(tool.name, prefixCounts);
+        const bucket = automatic.get(id) ?? [];
+        bucket.push(tool);
+        automatic.set(id, bucket);
+    }
+    for (const [id, members] of automatic) {
+        const safeId = groups.has(id) ? `auto:${id}` : id;
+        groups.set(safeId, createGroup(safeId, members.length > 1 ? `${id} tool family` : members[0].description, [], members));
+    }
+    const toolToGroup = new Map();
+    for (const group of groups.values()) {
+        for (const tool of group.tools)
+            toolToGroup.set(tool.name, group.id);
+    }
+    return {
+        tools,
+        groups,
+        toolToGroup,
+        totalEstimatedTokens: [...tools.values()].reduce((total, tool) => total + tool.estimatedTokens, 0),
+    };
+}
+function scoreGroup(group, query) {
+    const normalizedQuery = normalize(query);
+    if (normalizedQuery === '')
+        return 0;
+    const queryTokens = tokens(query);
+    const labels = [group.id, ...group.aliases, ...group.tools.map(tool => tool.name)]
+        .map(normalize)
+        .filter(label => label !== '');
+    const groupTokens = new Set(tokens(`${group.id} ${group.aliases.join(' ')}`));
+    const toolNameTokens = new Set(group.tools.flatMap(tool => tokens(tool.name)));
+    const descriptionTokens = new Set(tokens(`${group.description} ${group.tools.map(tool => tool.description).join(' ')}`));
+    let score = 0;
+    if (normalize(group.id) === normalizedQuery)
+        score += 120;
+    if (group.aliases.some(alias => normalize(alias) === normalizedQuery))
+        score += 110;
+    if (group.tools.some(tool => normalize(tool.name) === normalizedQuery))
+        score += 100;
+    if (labels.some(label => normalizedQuery.includes(label)))
+        score += 36;
+    if (group.searchText.includes(normalizedQuery))
+        score += 24;
+    for (const token of queryTokens) {
+        if (groupTokens.has(token))
+            score += 20;
+        if (toolNameTokens.has(token))
+            score += 12;
+        if (descriptionTokens.has(token))
+            score += 4;
+    }
+    return score;
+}
+export function searchCatalog(catalog, query, limit) {
+    return [...catalog.groups.values()]
+        .map(group => ({ group, score: scoreGroup(group, query) }))
+        .filter(candidate => candidate.score > 0)
+        .sort((left, right) => right.score - left.score
+        || left.group.estimatedTokens - right.group.estimatedTokens
+        || left.group.id.localeCompare(right.group.id))
+        .slice(0, limit)
+        .map(({ group, score }) => ({
+        group: group.id,
+        description: group.description.slice(0, 180),
+        score,
+        estimatedTokens: group.estimatedTokens,
+        tools: group.tools.map(tool => tool.name),
+    }));
+}
+function termFrequency(documentTokens, term) {
+    return documentTokens.reduce((count, token) => count + (token === term ? 1 : 0), 0);
+}
+/**
+ * Rank individual tools with exact-name bonuses and a compact BM25-style score.
+ * Definitions, parameter descriptions, enums, and nested property names all
+ * participate in the searchable document.
+ */
+export function searchTools(catalog, query, limit) {
+    const normalizedQuery = normalize(query);
+    const queryTokens = [...new Set(tokens(query))];
+    if (normalizedQuery === '' || queryTokens.length === 0)
+        return [];
+    const documents = [...catalog.tools.values()].map(tool => ({
+        tool,
+        group: catalog.groups.get(catalog.toolToGroup.get(tool.name) ?? ''),
+        text: '',
+        tokens: [],
+    }));
+    for (const document of documents) {
+        const groupText = document.group === undefined
+            ? ''
+            : `${document.group.id} ${document.group.description} ${document.group.aliases.join(' ')}`;
+        document.text = normalize(`${document.tool.searchText} ${groupText}`);
+        document.tokens = tokens(document.text);
+    }
+    const averageLength = documents.length === 0
+        ? 1
+        : documents.reduce((total, document) => total + document.tokens.length, 0) / documents.length;
+    const k1 = 1.2;
+    const b = 0.75;
+    return documents
+        .map(({ tool, group, text, tokens: documentTokens }) => {
+        let score = 0;
+        const normalizedName = normalize(tool.name);
+        if (normalizedName === normalizedQuery)
+            score += 160;
+        else if (normalizedName.includes(normalizedQuery))
+            score += 48;
+        if (text.includes(normalizedQuery))
+            score += 20;
+        const labels = [group?.id ?? '', ...group?.aliases ?? []]
+            .map(normalize)
+            .filter(label => label !== '');
+        if (labels.some(label => normalizedQuery.includes(label)))
+            score += 40;
+        for (const term of queryTokens) {
+            const frequency = termFrequency(documentTokens, term);
+            if (frequency === 0)
+                continue;
+            const documentFrequency = documents.reduce((count, document) => count + (document.tokens.includes(term) ? 1 : 0), 0);
+            const inverseFrequency = Math.log(1 + (documents.length - documentFrequency + 0.5) / (documentFrequency + 0.5));
+            const denominator = frequency + k1 * (1 - b + b * documentTokens.length / averageLength);
+            score += inverseFrequency * (frequency * (k1 + 1)) / denominator * 10;
+            if (tokens(tool.name).includes(term))
+                score += 18;
+        }
+        return {
+            tool,
+            score,
+            group: catalog.toolToGroup.get(tool.name) ?? tool.name,
+        };
+    })
+        .filter(candidate => candidate.score > 0)
+        .sort((left, right) => right.score - left.score
+        || left.tool.estimatedTokens - right.tool.estimatedTokens
+        || left.tool.name.localeCompare(right.tool.name))
+        .slice(0, limit)
+        .map(({ tool, score, group }) => ({
+        name: tool.name,
+        description: tool.description,
+        parameters: tool.parameters,
+        group,
+        score: Math.round(score * 100) / 100,
+        estimatedTokens: tool.estimatedTokens,
+        groupTools: catalog.groups.get(group)?.tools.map(member => member.name) ?? [tool.name],
+    }));
+}
+export function matchingToolNames(catalog, patterns) {
+    return [...catalog.tools.keys()].filter(name => matchesToolName(name, patterns));
+}

package/lib/defaults.d.ts

@@ -0,0 +1,16 @@
+import type { SkillBindingConfig, ToolGroupConfig } from './types.js';
+export declare const DEFAULT_MODE: "stable-proxy";
+export declare const DEFAULT_TOOL_NAME = "tool_search";
+export declare const DEFAULT_DISPATCH_TOOL_NAME = "tool_dispatch";
+export declare const DEFAULT_MAX_RESULTS = 5;
+export declare const DEFAULT_ACTIVATION_GROUP_LIMIT = 1;
+export declare const DEFAULT_MAX_ACTIVE_GROUPS = 3;
+export declare const DEFAULT_MAX_ACTIVE_TOOL_TOKENS = 6000;
+export declare const DEFAULT_RETENTION_TURNS = 6;
+export declare const DEFAULT_CHARACTERS_PER_TOKEN = 4;
+export declare const DEFAULT_REQUIRE_DISCOVERY = true;
+export declare const DEFAULT_STATUS_GRANTS_DISCOVERY = false;
+export declare const DEFAULT_DEFER_TOOL_GUIDANCE = true;
+export declare const DEFAULT_ALWAYS_VISIBLE: readonly ["skill", "ask_user_question", "read", "write", "edit", "glob", "grep", "bash", "todo_write", "dsh_im_return_file", "report", "submit_*", "structured_output*"];
+export declare const DEFAULT_SKILL_BINDINGS: readonly SkillBindingConfig[];
+export declare const DEFAULT_GROUPS: readonly ToolGroupConfig[];

package/lib/defaults.js

@@ -0,0 +1,122 @@
+export const DEFAULT_MODE = 'stable-proxy';
+export const DEFAULT_TOOL_NAME = 'tool_search';
+export const DEFAULT_DISPATCH_TOOL_NAME = 'tool_dispatch';
+export const DEFAULT_MAX_RESULTS = 5;
+export const DEFAULT_ACTIVATION_GROUP_LIMIT = 1;
+export const DEFAULT_MAX_ACTIVE_GROUPS = 3;
+export const DEFAULT_MAX_ACTIVE_TOOL_TOKENS = 6_000;
+export const DEFAULT_RETENTION_TURNS = 6;
+export const DEFAULT_CHARACTERS_PER_TOKEN = 4;
+export const DEFAULT_REQUIRE_DISCOVERY = true;
+export const DEFAULT_STATUS_GRANTS_DISCOVERY = false;
+export const DEFAULT_DEFER_TOOL_GUIDANCE = true;
+export const DEFAULT_ALWAYS_VISIBLE = [
+    'skill',
+    'ask_user_question',
+    'read',
+    'write',
+    'edit',
+    'glob',
+    'grep',
+    'bash',
+    'todo_write',
+    'dsh_im_return_file',
+    'report',
+    'submit_*',
+    'structured_output*',
+];
+export const DEFAULT_SKILL_BINDINGS = [];
+export const DEFAULT_GROUPS = [
+    {
+        id: 'browser',
+        description: 'Interactive browser navigation, page inspection, clicks, forms, screenshots, and tabs.',
+        aliases: ['browser', 'web page', '网页', '浏览器'],
+        include: ['browser_*', 'navigate', 'click', 'screenshot', 'computer_*'],
+    },
+    {
+        id: 'vision',
+        description: 'Image understanding, OCR, comparison, annotation, and visual routing.',
+        aliases: ['vision', 'image analysis', '图片', '视觉', '识图'],
+        include: ['vision_*', 'ocr*', 'analyze_image*', 'compare_image*', 'describe_image*'],
+    },
+    {
+        id: 'image-generation',
+        description: 'Image generation and image editing.',
+        aliases: ['image generation', 'generate image', '生图', '图片生成'],
+        include: [
+            'imagegen*',
+            'image_gen*',
+            'generate_image*',
+            'edit_image*',
+            'get_image_generation*',
+            'cancel_image_generation*',
+            'image_generation*',
+        ],
+    },
+    {
+        id: 'filesystem',
+        description: 'Read, search, create, and edit files and directories.',
+        aliases: ['files', 'filesystem', 'code editing', '文件', '代码编辑'],
+        include: ['read', 'write', 'edit', 'glob', 'grep', 'read_file*', 'write_file*', 'list_dir*', 'search_file*', 'str_replace*', 'fs_*'],
+    },
+    {
+        id: 'terminal',
+        description: 'Shell commands, terminal sessions, jobs, and local process control.',
+        aliases: ['terminal', 'shell', 'command', '终端', '命令行'],
+        include: ['bash', 'pwsh', 'shell_*', 'terminal_*', 'exec_*', 'job_*'],
+    },
+    {
+        id: 'web',
+        description: 'Web search, HTTP fetching, and URL content retrieval.',
+        aliases: ['web search', 'internet', 'http', '联网', '网页搜索'],
+        include: ['web_*', 'http_*', 'fetch*', 'search_web*'],
+    },
+    {
+        id: 'database',
+        description: 'Database inspection, queries, migrations, and records.',
+        aliases: ['database', 'sql', '数据库'],
+        include: ['database_*', 'db_*', 'sql_*', 'query_*'],
+    },
+    {
+        id: 'remote-ops',
+        description: 'SSH, SFTP, tunnels, and remote host operations.',
+        aliases: ['ssh', 'remote', 'server', '远程', '服务器'],
+        include: ['ssh_*', 'sftp_*', 'tunnel_*', 'remote_*'],
+    },
+    {
+        id: 'memory',
+        description: 'Durable memory, recall, notes, and knowledge retrieval.',
+        aliases: ['memory', 'recall', '记忆', '知识库'],
+        include: ['memory_*', 'recall*', 'mneme_*', 'note_*'],
+    },
+    {
+        id: 'workbench',
+        description: 'Workspace workbench, artifacts, previews, and project utilities.',
+        aliases: ['workbench', 'workspace', '工作台'],
+        include: ['workbench_*', 'workspace_*', 'artifact_*', 'preview_*'],
+    },
+    {
+        id: 'agent-teams',
+        description: 'Team members, delegated tasks, messages, and coordination.',
+        aliases: ['team', 'delegate', '协作', '团队', '子任务'],
+        include: ['team_*', 'agent_team*', 'spawn_agent*', 'send_message*', 'followup_task*', 'wait_agent*', 'list_agent*'],
+    },
+    {
+        id: 'subagents',
+        description: 'Subagent spawning, continuation, inspection, and collection.',
+        aliases: ['subagent', 'delegate', '子智能体', '委派'],
+        include: ['subagent*', 'spawn*', 'fork_agent*', 'continue_agent*'],
+    },
+    {
+        id: 'workflow',
+        description: 'Workflow definitions, runs, schedules, and automation control.',
+        aliases: ['workflow', 'automation', 'schedule', '工作流', '自动化'],
+        include: ['workflow_*', 'schedule_*', 'automation_*', 'cron_*'],
+    },
+    {
+        id: 'interface',
+        description: 'Generated interface components, interactive artifacts, and structured UI.',
+        aliases: ['interface', 'ui', '组件', '界面'],
+        include: ['genui_*', 'ui_*', 'component_*'],
+    },
+];

package/lib/index.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Cache-stable progressive disclosure for the DeepSeek Harness tool registry.
+ *
+ * The default mode keeps one byte-stable model-facing surface and dispatches
+ * deferred tools through the ordinary Harness execution pipeline. A legacy
+ * dynamic mode remains available for deployments that require native schemas.
+ */
+import type { Context } from '@deepseek-ai/cordis';
+import z from '@deepseek-ai/schemastery';
+import type { ProgressiveMode, ResolvedConfig, SkillBindingConfig, ToolGroupConfig } from './types.js';
+export { buildCatalog, estimateSchemaTokens, searchCatalog, searchTools } from './catalog.js';
+export { activateGroups, createProgressiveState, expireGroups, proposeSearch, restoreSnapshot, snapshotState, touchTool, } from './state.js';
+export type { ActiveGroupState, CatalogTool, DeferredGroupSummary, DeferredToolMatch, ProgressiveMode, ProxySearchResultValue, ResolvedConfig, SearchMatch, SearchResultValue, SkillBindingConfig, StateSnapshot, ToolCatalog, ToolGroup, ToolGroupConfig, ToolSchemaView, } from './types.js';
+export declare const name = "tokens-progressive-tools";
+export declare const inject: string[];
+export interface Config {
+    /** Stable proxy is cache-friendly; dynamic exposes changing native families. */
+    readonly mode?: ProgressiveMode;
+    /** Registered discovery tool name. */
+    readonly toolName?: string;
+    /** Registered stable dispatcher name. */
+    readonly dispatchToolName?: string;
+    /** Tool-name wildcard patterns that stay directly visible. */
+    readonly alwaysVisible?: readonly string[];
+    /** Ordered family rules. The first matching family owns a tool. */
+    readonly groups?: readonly ToolGroupConfig[];
+    /** Successful skill calls that make bound tools dispatchable. */
+    readonly skillBindings?: readonly SkillBindingConfig[];
+    /** Maximum exact search matches returned to the caller. */
+    readonly maxResults?: number;
+    /** Highest-ranked families activated by one dynamic-mode search. */
+    readonly activationGroupLimit?: number;
+    /** Maximum retained active families in dynamic mode. */
+    readonly maxActiveGroups?: number;
+    /** Approximate schema-token budget for active dynamic-mode families. */
+    readonly maxActiveToolTokens?: number;
+    /** Dynamic-mode inactivity turns before expiry; zero disables expiry. */
+    readonly retentionTurns?: number;
+    /** Schema characters represented by one estimated token. */
+    readonly charactersPerToken?: number;
+    /** Require a successful search or skill binding before proxy dispatch. */
+    readonly requireDiscovery?: boolean;
+    /** Let one status listing make every cataloged name dispatchable. */
+    readonly statusGrantsDiscovery?: boolean;
+    /** Remove exact hidden tool guidance sections from the stable prompt. */
+    readonly deferToolGuidance?: boolean;
+}
+export declare const Config: z<Config>;
+export declare function resolveConfig(config?: Config): ResolvedConfig;
+export declare function apply(ctx: Context, input: Config): void;