@preply/ds-ai-core 11.1.0

package/src/tools/search-token/context/__tests__/get-tokens-data.test.ts

@@ -0,0 +1,148 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const mocks = vi.hoisted(() => ({
+    tokens: {
+        spacing: {
+            '4': ['base', 'abc123', false],
+            '8': ['base', 'def456', false],
+        },
+        root: {
+            radius: {
+                xs: ['base', 'ghi789', false],
+            },
+        },
+        color: {
+            text: {
+                primary: ['base', 'aaa111', true],
+            },
+        },
+    } as unknown as typeof import('@preply/ds-core').tokens,
+    valuesWeb: {
+        tabc123: '4px',
+        tdef456: '8px',
+        tghi789: 2,
+        taaa111: { base: '#111111' },
+    } as unknown as typeof import('@preply/ds-theme-tokyo-ui').VALUES_WEB,
+    valuesRn: {
+        tabc123: 4,
+        tdef456: 8,
+        tghi789: 2,
+        taaa111: { base: '#101010' },
+    } as unknown as typeof import('@preply/ds-theme-tokyo-ui').VALUES_RN,
+}));
+
+vi.mock(import('@preply/ds-theme-tokyo-ui'), async () => ({
+    VALUES_WEB: mocks.valuesWeb,
+    VALUES_RN: mocks.valuesRn,
+}));
+
+vi.mock(import('@preply/ds-core'), async () => ({
+    ...(await import('@preply/ds-core')),
+    tokens: mocks.tokens,
+}));
+
+beforeEach(() => {
+    vi.resetModules();
+    vi.resetAllMocks();
+});
+
+describe('getTokensData', () => {
+    it('flattens token tree and computes formatted names for web', async () => {
+        const { getTokensData } = await import('../get-tokens-data');
+        const data = getTokensData();
+
+        expect(data.web).toMatchInlineSnapshot(`
+          [
+            {
+              "less": "@spacing-4",
+              "name": "spacing.4",
+              "private": false,
+              "scss": "$spacing-4",
+              "typescript": "spacing[4]",
+              "value": "4px",
+            },
+            {
+              "less": "@spacing-8",
+              "name": "spacing.8",
+              "private": false,
+              "scss": "$spacing-8",
+              "typescript": "spacing[8]",
+              "value": "8px",
+            },
+            {
+              "less": "@root-radius-xs",
+              "name": "root.radius.xs",
+              "private": true,
+              "scss": "$root-radius-xs",
+              "typescript": "root.radius.xs",
+              "value": 2,
+            },
+            {
+              "less": "@color-text-primary",
+              "name": "color.text.primary",
+              "private": false,
+              "scss": "$color-text-primary",
+              "typescript": "color.text.primary",
+              "value": "#111111",
+            },
+          ]
+        `);
+    });
+
+    it('resolves react-native values using VALUES_RN map', async () => {
+        const { getTokensData } = await import('../get-tokens-data');
+        const data = getTokensData();
+
+        expect(data['react-native']).toMatchInlineSnapshot(`
+          [
+            {
+              "less": "@spacing-4",
+              "name": "spacing.4",
+              "private": false,
+              "scss": "$spacing-4",
+              "typescript": "spacing[4]",
+              "value": 4,
+            },
+            {
+              "less": "@spacing-8",
+              "name": "spacing.8",
+              "private": false,
+              "scss": "$spacing-8",
+              "typescript": "spacing[8]",
+              "value": 8,
+            },
+            {
+              "less": "@root-radius-xs",
+              "name": "root.radius.xs",
+              "private": true,
+              "scss": "$root-radius-xs",
+              "typescript": "root.radius.xs",
+              "value": 2,
+            },
+            {
+              "less": "@color-text-primary",
+              "name": "color.text.primary",
+              "private": false,
+              "scss": "$color-text-primary",
+              "typescript": "color.text.primary",
+              "value": "#101010",
+            },
+          ]
+        `);
+    });
+
+    it('throws when token value cannot be resolved', async () => {
+        vi.doMock('@preply/ds-core', () => ({
+            isToken: (value: unknown) => Array.isArray(value),
+            tokens: {
+                spacing: {
+                    '16': ['base', 'missing', false],
+                },
+            },
+        }));
+
+        const { getTokensData } = await import('../get-tokens-data');
+
+        expect(() => getTokensData()).toThrow('Token value not found for token id: missing');
+    });
+});

package/src/tools/search-token/context/get-tokens-data.ts

@@ -0,0 +1,103 @@
+import { isToken, tokens } from '@preply/ds-core';
+import { VALUES_RN, VALUES_WEB } from '@preply/ds-theme-tokyo-ui';
+import { ThemeTokenValue, Token, TOKEN_ID_FIELD } from '@preply/ds-core-types';
+import { Platform } from '../../../types';
+
+const PRIVATE_TOKEN_PATTERNS = ['dsInternalPrimitive', 'root'];
+
+/** @example formatScssName('dropShadow.1.shorthand') // '$drop-shadow-1-shorthand' */
+const formatScssName = (name: string) => '$' + name.replace(/\./g, '-');
+/** @example formatLessName('dropShadow.1.shorthand') // '@drop-shadow-1-shorthand' */
+const formatLessName = (name: string) => '@' + name.replace(/\./g, '-');
+/** @example formatTypescriptName('dropShadow.1.shorthand') // 'dropShadow[1].shorthand' */
+const formatTypescriptName = (name: string) =>
+    name
+        .split('.')
+        .map(part => {
+            const isNumeric = !isNaN(Number(part));
+            const hasHyphen = part.includes('-');
+            if (isNumeric || hasHyphen) return `[${part}]`;
+            return part;
+        })
+        .join('.')
+        .replace(/\.\[/g, '[');
+
+/**
+ * Resolves a token's numeric ID to its concrete theme value.
+ * Theme values are keyed as `t{id}` and can be:
+ *  - a primitive (string | number) -> returned directly
+ *  - an object with a `.base` key  -> `.base` is the canonical value (responsive tokens)
+ *  - an object without `.base`     -> first value is used as fallback (e.g. platform-specific)
+ */
+function getTokenValue(token: Token, values: Record<string, ThemeTokenValue>): string | number {
+    const tokenId = token[TOKEN_ID_FIELD];
+    const value = values[`t${tokenId}`];
+    if (value === undefined) {
+        throw new Error(`Token value not found for token id: ${tokenId}`);
+    }
+
+    if (typeof value === 'object') {
+        const base = (value as Record<string, string | number>).base;
+        if (base !== undefined) return base;
+
+        const firstValue = Object.values(value)[0];
+        if (firstValue !== undefined) return firstValue as string | number;
+    }
+
+    return value as string | number;
+}
+
+/** Recursively walks the nested token tree, collecting leaf tokens (identified by `isToken`) into a flat array. */
+function flattenTokens(
+    input: unknown,
+    values: Record<string, ThemeTokenValue>,
+    path: string[] = [],
+    output: TokenData[] = [],
+): TokenData[] {
+    if (isToken(input)) {
+        const name = path.join('.');
+        const value = getTokenValue(input, values);
+        output.push({
+            name,
+            scss: formatScssName(name),
+            less: formatLessName(name),
+            typescript: formatTypescriptName(name),
+            value,
+            private: PRIVATE_TOKEN_PATTERNS.some(pattern => name.startsWith(pattern)),
+        });
+        return output;
+    }
+
+    if (!input || typeof input !== 'object') {
+        return output;
+    }
+
+    for (const [key, value] of Object.entries(input as Record<string, unknown>)) {
+        flattenTokens(value, values, [...path, key], output);
+    }
+
+    return output;
+}
+
+export type TokenData = {
+    name: string;
+    scss: string;
+    less: string;
+    typescript: string;
+    value: string | number;
+    private: boolean;
+};
+
+type PlatformTokens = Record<Platform, TokenData[]>;
+
+export function getTokensData(): PlatformTokens {
+    const valuesByPlatform: Record<Platform, Record<string, ThemeTokenValue>> = {
+        web: VALUES_WEB as Record<string, ThemeTokenValue>,
+        'react-native': VALUES_RN as Record<string, ThemeTokenValue>,
+    };
+
+    return {
+        web: flattenTokens(tokens, valuesByPlatform.web),
+        'react-native': flattenTokens(tokens, valuesByPlatform['react-native']),
+    };
+}

package/src/tools/search-token/context/index.ts

@@ -0,0 +1,4 @@
+// Build-time macro: getTokensData() runs at compile time and its return value is inlined into the bundle
+import { getTokensData } from './get-tokens-data' with { type: 'macro' };
+
+export default getTokensData();

package/src/tools/search-token/search-token-by-name.ts

@@ -0,0 +1,110 @@
+import { createSearchIndex } from '../../utils/create-search-index';
+import { PLATFORMS } from '../../types';
+import dedent from 'dedent';
+import { defineTool } from '../../utils/define-tool';
+import * as z from 'zod';
+import { stringify } from 'yaml';
+import { formatList } from '../../utils/format-list';
+import tokens from './context';
+import { Format, formatToken, getExample } from './token-utils';
+
+export type { Format };
+
+/**
+ * Normalizes token name to the unified format, so queries in all formats will match the same token:
+ * - dropShadow.1.shorthand
+ * - $dropShadow-1-shorthand
+ * - @dropShadow-1-shorthand
+ * - dropShadow[1].shorthand
+ */
+const normalize = (term: string) =>
+    term
+        .replace(/[^a-zA-Z0-9]+/g, ' ')
+        .trim()
+        .toLowerCase();
+
+const indexes = {
+    web: createSearchIndex(tokens.web, { ref: 'name', normalize }),
+    'react-native': createSearchIndex(tokens['react-native'], { ref: 'name', normalize }),
+};
+
+export const searchTokenByName = defineTool({
+    name: 'search_token_by_name',
+    description: 'Search for a design system style tokens by name',
+    arguments: z.object({
+        name: z.string().describe(
+            dedent`
+                The token name to search for. This can be in any format:
+                - raw token name (e.g. dropShadow.1.shorthand)
+                - scss (e.g. $dropShadow-1-shorthand)
+                - less (e.g. @dropShadow-1-shorthand)
+                - typescript (e.g. dropShadow[1].shorthand)
+            `,
+        ),
+    }),
+    options: z.object({
+        platform: z.enum(PLATFORMS).describe('Target platform for token values. Required.'),
+        format: z.enum(['scss', 'less', 'typescript']).optional().default('scss').describe(dedent`
+                Optional output format for web platform only. Defaults to "scss".
+                - "scss" - SCSS variable (e.g. $drop-shadow-1-shorthand)
+                - "less" - Less variable (e.g. @drop-shadow-1-shorthand)
+                - "typescript" - Typescript variable for usage in styled-components (e.g. dropShadow[1].shorthand)
+
+                For "react-native" platform this option is ignored.
+            `),
+        includeExample: z
+            .boolean()
+            .default(true)
+            .describe('Whether to include an usage example. Default: true'),
+        includePrivate: z.boolean().default(false).describe(dedent`
+                Whether to include private tokens. Default: false
+
+                Private tokens starts from "dsInternalPrimitive" or "root" prefixes.
+                They're not intended for public use, however they're still available in the system.
+                Only use this option if you wasn't able to find public token or you're confident that private token is what you need.
+            `),
+    }),
+    callback: ({ name, platform, format = 'scss', includeExample, includePrivate }) => {
+        const outputFormat: Format = platform === 'web' ? format : 'scss';
+        const matches = indexes[platform]
+            .search(name, 10)
+            .filter(m => includePrivate || !m.item.private);
+
+        if (!matches.length) {
+            return [`No results found for "${name}"`];
+        }
+
+        const [first, ...rest] = matches;
+
+        const examples = includeExample
+            ? [
+                  platform === 'web'
+                      ? `Here is an example of how to use design-system tokens in ${outputFormat}:`
+                      : 'Here is an example of how to use design-system tokens in react-native:',
+                  getExample(platform, outputFormat),
+              ]
+            : [];
+
+        if (!first.exact) {
+            return [
+                `There is no exact match for "${name}", but here are some similar tokens:`,
+                formatList(matches.map(m => formatToken(m.item, platform, outputFormat))),
+                ...examples,
+            ];
+        }
+
+        const similarMessages = rest.length
+            ? [
+                  'Also, here you can find some tokens with similar names:',
+                  formatList(rest.map(m => formatToken(m.item, platform, outputFormat))),
+              ]
+            : [];
+
+        return [
+            `Found exact match for "${name}":`,
+            stringify(formatToken(first.item, platform, outputFormat)).trim(),
+            ...examples,
+            ...similarMessages,
+        ];
+    },
+});

package/src/tools/search-token/search-token-by-value.ts

@@ -0,0 +1,107 @@
+// Build-time macro: getTokensData() runs at compile time and its return value is inlined into the bundle
+import { TokenData, getTokensData } from './context/get-tokens-data' with { type: 'macro' };
+import { PLATFORMS, Platform } from '../../types';
+import dedent from 'dedent';
+import { defineTool } from '../../utils/define-tool';
+import * as z from 'zod';
+import { formatList } from '../../utils/format-list';
+import { Format, formatToken, getExample } from './token-utils';
+
+const allTokens = getTokensData();
+
+function buildValueMap(tokens: TokenData[]): Record<string, TokenData[]> {
+    const map: Record<string, TokenData[]> = {};
+    for (const token of tokens) {
+        const key = token.value.toString().toLowerCase();
+        map[key] = map[key] ?? [];
+        map[key].push(token);
+    }
+    return map;
+}
+
+const valueMaps = {
+    web: buildValueMap(allTokens.web),
+    'react-native': buildValueMap(allTokens['react-native']),
+};
+
+function searchByValue(platform: Platform, value: string | number, includePrivate = false) {
+    const map = valueMaps[platform];
+    const normalizedValue = value.toString().toLowerCase();
+    let results = map[normalizedValue] ?? [];
+
+    if (!isNaN(Number(value))) {
+        const pxResults = map[`${normalizedValue}px`] ?? [];
+        const emResults = map[`${normalizedValue}em`] ?? [];
+        results = [...results, ...pxResults, ...emResults];
+    }
+
+    if (!includePrivate) {
+        results = results.filter(t => !t.private);
+    }
+
+    return [...new Set(results)];
+}
+
+export const searchTokenByValue = defineTool({
+    name: 'search_token_by_value',
+    description: 'Search for a design system style tokens by its value',
+    arguments: z.object({
+        value: z.string().describe(
+            dedent`
+                The token value to search for. This can be:
+                - numeric values (e.g. 500)
+                - colors (e.g. #000000 or rgba(0, 0, 0, 0.5))
+                - sizes (e.g. 16px or 2em)
+                - fonts (e.g. Inter or Arial)
+                - etc.
+            `,
+        ),
+    }),
+    options: z.object({
+        platform: z.enum(PLATFORMS).describe('Target platform for token values. Required.'),
+        format: z.enum(['scss', 'less', 'typescript']).optional().default('scss').describe(dedent`
+                Optional output format for web platform only. Defaults to "scss".
+                - "scss" - SCSS variable (e.g. $drop-shadow-1-shorthand)
+                - "less" - Less variable (e.g. @drop-shadow-1-shorthand)
+                - "typescript" - Typescript variable for usage in styled-components (e.g. dropShadow[1].shorthand)
+
+                For "react-native" platform this option is ignored.
+            `),
+        includeExample: z
+            .boolean()
+            .default(true)
+            .describe('Whether to include an usage example. Default: true'),
+        includePrivate: z.boolean().default(false).describe(dedent`
+                Whether to include private tokens. Default: false
+
+                Private tokens starts from "dsInternalPrimitive" or "root" prefixes.
+                They're not intended for public use, however they're still available in the system.
+                Only use this option if you wasn't able to find public token or you're confident that private token is what you need.
+            `),
+    }),
+    callback: ({ value, platform, format = 'scss', includeExample, includePrivate }) => {
+        const outputFormat: Format = platform === 'web' ? format : 'scss';
+        const results = searchByValue(platform, value, includePrivate);
+
+        if (!results.length) {
+            return [`No results found for "${value}"`];
+        }
+
+        const formattedResults = results.map(r => formatToken(r, platform, outputFormat));
+
+        const examples = includeExample
+            ? [
+                  platform === 'web'
+                      ? `Here is an example of how to use design-system tokens in ${outputFormat}:`
+                      : 'Here is an example of how to use design-system tokens in react-native:',
+                  getExample(platform, outputFormat),
+              ]
+            : [];
+
+        return [
+            `Found ${results.length} tokens with value "${value}"`,
+            formatList(formattedResults),
+            ...examples,
+        ];
+    },
+});

package/src/tools/search-token/token-utils.ts

@@ -0,0 +1,60 @@
+import type { TokenData } from './context/get-tokens-data';
+import { Platform } from '../../types';
+import dedent from 'dedent';
+
+export type Format = 'scss' | 'less' | 'typescript';
+
+export function formatToken(token: TokenData, platform: Platform, format: Format) {
+    return {
+        token: platform === 'web' ? token[format] : token.typescript,
+        value: token.value,
+    };
+}
+
+const webExamples: Record<Format, string> = {
+    scss: dedent`
+        @use '@preply/ds-web-core/dist/generated/tokens' as *;
+
+        .MyComponent {
+            box-shadow: $dropShadow-1-shorthand;
+        }
+    `,
+    less: dedent`
+        @import '~@preply/ds-web-core/dist/generated/tokens.less';
+
+        .MyComponent {
+            box-shadow: @dropShadow-1-shorthand;
+        }
+    `,
+    typescript: dedent`
+        import styled from 'styled-components';
+        import { dropShadow } from '@preply/ds-core';
+        import { getTokenVar } from '@preply/ds-web-core';
+
+        const MyComponent = styled.div\`
+            box-shadow: \${getTokenVar(dropShadow[1].shorthand)};
+        \`;
+    `,
+};
+
+const reactNativeExample = dedent`
+    import { View } from 'react-native';
+    import { color } from '@preply/ds-core';
+    import { createStyleSheet, useStyleSheet } from '@preply/ds-rn-core';
+
+    const rawStyles = createStyleSheet({
+        container: {
+            backgroundColor: color.background.primary,
+        },
+    });
+
+    export const MyComponent = () => {
+        const styles = useStyleSheet(rawStyles);
+
+        return <View style={styles.container} />;
+    };
+`;
+
+export function getExample(platform: Platform, format: Format) {
+    return platform === 'web' ? webExamples[format] : reactNativeExample;
+}

package/src/types.ts

@@ -0,0 +1,3 @@
+export type Platform = 'web' | 'react-native';
+
+export const PLATFORMS = ['web', 'react-native'] as const satisfies Platform[];

package/src/utils/create-search-index.ts

@@ -0,0 +1,121 @@
+/* eslint-disable import/no-named-as-default-member */
+import lunr from 'lunr';
+
+// Lunr requires globally unique names for registered pipeline functions.
+// This counter ensures each createSearchIndex call gets a distinct function name.
+let pipelineCounter = 0;
+
+type Result<T> = {
+    item: T;
+    /** Whether the query matched a ref exactly (as opposed to a fuzzy/partial match). */
+    exact: boolean;
+};
+
+/**
+ * Extracts only the keys of T whose values are strings.
+ * Used to constrain `ref` so it always points to a string field.
+ */
+type StringKeys<T extends Record<string, unknown>, K = keyof T> = K extends string
+    ? T[K] extends string
+        ? K
+        : never
+    : never;
+
+/**
+ * Builds a lunr full-text search index over `items` and returns a thin wrapper
+ * with `search` (ranked results) and `get` (exact lookup) methods.
+ *
+ * @param items   - The dataset to index.
+ * @param ref     - Which string field to use as the unique document key.
+ * @param normalize - Optional transform applied to both stored refs and incoming
+ *                    queries so that e.g. case or naming-convention differences
+ *                    don't prevent matches. When provided, a custom pipeline
+ *                    function is also registered to split compound tokens.
+ */
+export function createSearchIndex<T extends Record<string, unknown>>(
+    /** The dataset to index. */
+    items: T[],
+    options: {
+        /** The field to use as the unique document key. */
+        ref: StringKeys<T>;
+        /** An optional transform applied to both stored refs and incoming queries */
+        normalize?: (term: string) => string;
+    },
+) {
+    const { ref, normalize = s => s } = options;
+
+    const itemsByRef = new Map<string, T>();
+    for (const item of items) {
+        itemsByRef.set(normalize(item[ref] as string), item);
+    }
+
+    // When a normalize function is provided we also register a custom lunr
+    // pipeline step that splits compound tokens before stemming.
+    let pipelineFn: lunr.PipelineFunction | undefined;
+    if (options.normalize) {
+        const fnName = `createSearchIndex_normalize_${pipelineCounter++}`;
+        pipelineFn = (token: lunr.Token) => {
+            const tokenParts = token
+                .toString()
+                .replace(/[^a-zA-Z0-9]/g, ' ')
+                .trim()
+                .split(/\s+/);
+
+            return tokenParts.map(part => token.clone().update(() => part));
+        };
+        // Pipeline functions must be registered globally before they can be
+        // added to an index pipeline; the unique fnName avoids collisions.
+        lunr.Pipeline.registerFunction(pipelineFn, fnName);
+    }
+
+    // Build the lunr index. `__ref` is an internal field that holds the
+    // normalized ref — we use a synthetic name to avoid clashing with any
+    // field already present on the items.
+    const index = lunr(function () {
+        this.ref('__ref');
+        this.field('__ref');
+
+        if (pipelineFn) {
+            this.pipeline.before(lunr.stemmer, pipelineFn);
+            this.searchPipeline.before(lunr.stemmer, pipelineFn);
+        }
+
+        for (const item of items) {
+            this.add({
+                ...item,
+                __ref: normalize(item[ref] as string),
+            });
+        }
+    });
+
+    return {
+        /**
+         * Performs a full-text search over the indexed items.
+         *
+         * @example
+         * ```ts
+         * const results = index.search('dropShadow-1-shorthand');
+         * console.log(results);
+         * // [
+         * //   { item: { name: 'dropShadow-1-shorthand' }, exact: true },
+         * //   { item: { name: 'dropShadow-1-shorthand' }, exact: false },
+         * // ]
+         * ```
+         */
+        search(query: string, limit?: number): Result<T>[] {
+            const results = index.search(normalize(query)).map(result => {
+                const item = itemsByRef.get(result.ref)!;
+                return {
+                    item,
+                    exact: result.ref === normalize(query),
+                };
+            });
+
+            return limit ? results.slice(0, limit) : results;
+        },
+        /** Direct lookup by normalized ref value. Returns undefined if not found. */
+        get(query: string): T | undefined {
+            return itemsByRef.get(normalize(query));
+        },
+    };
+}