intor-translator 1.4.9 → 1.4.11

package/dist/index.cjs

@@ -520,3 +520,4 @@ function renderRichMessage(message, renderer) {
 exports.Translator = ScopeTranslator;
 exports.parseRichMessage = parseRichMessage;
 exports.renderRichMessage = renderRichMessage;
+exports.tokenize = tokenize;

package/dist/index.d.cts

@@ -239,6 +239,46 @@ type LocalizedReplacement<ReplacementSchema, K extends string> = ReplacementSche
  */
 type ScopedReplacement<ReplacementSchema, PK extends string | undefined, K extends string> = LocalizedReplacement<ReplacementSchema, `${PK}.${K}`>;
 
+/** Semantic tag attributes map. */
+type Attributes = Record<string, string>;
+
+/**
+ * Generic rich tag map used when no schema is available.
+ *
+ * Acts as a safe fallback for dynamic or unknown rich tag shapes.
+ */
+type Rich = Record<string, Attributes>;
+/**
+ * Rich tag map resolved from a localized rich schema.
+ *
+ * - If the key exists in the schema, resolves to the declared tag map
+ * - Otherwise falls back to generic `Rich`
+ *
+ * @example
+ * ```ts
+ * type RichSchema = { "{locale}": { link: { a: { href: string } } } }
+ * LocalizedRich<RichSchema, "link"> // => { a: { href: string } }
+ * LocalizedRich<RichSchema, "missing">; // => Rich
+ * ```
+ */
+type LocalizedRich<RichSchema, K extends string> = RichSchema extends {
+    "{locale}": infer LM;
+} ? IsNever<AtPath<LM, K>> extends true ? Rich : AtPath<LM, K> extends MessageObject ? AtPath<LM, K> : Rich : Rich;
+/**
+ * Rich tag map resolved under a scoped prefix key.
+ *
+ * Internally composes the full dot-path (`"${PK}.${K}"`)
+ * and delegates resolution to `LocalizedRich`.
+ *
+ * @example
+ * ```ts
+ * type RichSchema = { "{locale}": { app: { link: { a: { href: string } } } } };
+ * ScopedRich<RichSchema, "app", "link">; // => { a: { href: string } }
+ * ScopedRich<RichSchema, "app", "missing">; // => Rich
+ * ```
+ */
+type ScopedRich<RichSchema, PK extends string | undefined, K extends string> = LocalizedRich<RichSchema, `${PK}.${K}`>;
+
 /**
  * Options for initializing a translator
  *
@@ -418,9 +458,6 @@ declare class ScopeTranslator<M extends LocaleMessages | unknown = unknown, Repl
     scoped<PK extends LocalizedPreKey<M> | undefined = undefined>(preKey?: PK): PK extends string ? ScopeTranslatorMethods<M, ReplacementSchema, PK> : ScopeTranslatorMethods<M, ReplacementSchema>;
 }
 
-/** Semantic tag attributes map. */
-type Attributes = Record<string, string>;
-
 /** Semantic node produced by the AST builder. */
 type ASTNode = TextNode | TagNode | RawNode;
 /** Plain text node in the semantic AST. */
@@ -441,6 +478,46 @@ interface RawNode {
     value: Exclude<MessageValue, string>;
 }
 
+/** Flat semantic token produced by the message tokenizer. */
+type Token = TextToken | TagOpenToken | TagCloseToken;
+/** Plain text segment in the message. */
+interface TextToken {
+    type: "text";
+    value: string;
+    position: number;
+}
+/** Opening semantic tag with parsed attributes. */
+interface TagOpenToken {
+    type: "tag-open";
+    name: string;
+    attributes: Attributes;
+    position: number;
+}
+/** Closing semantic tag (no attributes). */
+interface TagCloseToken {
+    type: "tag-close";
+    name: string;
+    position: number;
+}
+
+/**
+ * Tokenize a localized message string into semantic tokens.
+ *
+ * Supported syntax:
+ *   - <tag>text</tag>
+ *   - <tag key="value">text</tag>
+ *
+ * Notes:
+ * - Produces a flat token stream (nesting is handled in a later stage)
+ * - Tag names are identifier-like and case-sensitive
+ * - Attributes are strict key="value" pairs (double quotes only)
+ * - Variables are assumed to be interpolated beforehand
+ *
+ * This tokenizer is intentionally minimal and fail-closed:
+ * any unrecognized or partially valid syntax is treated as plain text.
+ */
+declare const tokenize: (message: string) => Token[];
+
 /**
  * Parse a rich message value into a semantic AST.
  *
@@ -490,4 +567,4 @@ interface Renderer<Output> {
  */
 declare function renderRichMessage<Output>(message: MessageValue, renderer: Renderer<Output>): Output[];
 
-export { type ASTNode, type AtPath, type Attributes, type FallbackLocalesMap, type FormatHandler, type GeneratePaths, type HandlerContext, type IfLocaleMessages, type IfMessageObject, type Key, type LoadingHandler, type Locale, type LocaleMessages, type LocalizedKey, type LocalizedPreKey, type LocalizedReplacement, type LocalizedValue, type MessageObject, type MessageValue, type MissingHandler, type PreKey, type Renderer, type Replacement, type ScopedKey, type ScopedReplacement, type ScopedValue, type TranslateConfig, type TranslateContext, type TranslateHandlers, type TranslateHook, ScopeTranslator as Translator, type ScopeTranslatorMethods as TranslatorMethods, type ScopeTranslatorOptions as TranslatorOptions, type TranslatorPlugin, type Value, parseRichMessage, renderRichMessage };
+export { type ASTNode, type AtPath, type Attributes, type FallbackLocalesMap, type FormatHandler, type GeneratePaths, type HandlerContext, type IfLocaleMessages, type IfMessageObject, type Key, type LoadingHandler, type Locale, type LocaleMessages, type LocalizedKey, type LocalizedPreKey, type LocalizedReplacement, type LocalizedRich, type LocalizedValue, type MessageObject, type MessageValue, type MissingHandler, type PreKey, type Renderer, type Replacement, type Rich, type ScopedKey, type ScopedReplacement, type ScopedRich, type ScopedValue, type Token, type TranslateConfig, type TranslateContext, type TranslateHandlers, type TranslateHook, ScopeTranslator as Translator, type ScopeTranslatorMethods as TranslatorMethods, type ScopeTranslatorOptions as TranslatorOptions, type TranslatorPlugin, type Value, parseRichMessage, renderRichMessage, tokenize };

package/dist/index.d.ts

@@ -239,6 +239,46 @@ type LocalizedReplacement<ReplacementSchema, K extends string> = ReplacementSche
  */
 type ScopedReplacement<ReplacementSchema, PK extends string | undefined, K extends string> = LocalizedReplacement<ReplacementSchema, `${PK}.${K}`>;
 
+/** Semantic tag attributes map. */
+type Attributes = Record<string, string>;
+
+/**
+ * Generic rich tag map used when no schema is available.
+ *
+ * Acts as a safe fallback for dynamic or unknown rich tag shapes.
+ */
+type Rich = Record<string, Attributes>;
+/**
+ * Rich tag map resolved from a localized rich schema.
+ *
+ * - If the key exists in the schema, resolves to the declared tag map
+ * - Otherwise falls back to generic `Rich`
+ *
+ * @example
+ * ```ts
+ * type RichSchema = { "{locale}": { link: { a: { href: string } } } }
+ * LocalizedRich<RichSchema, "link"> // => { a: { href: string } }
+ * LocalizedRich<RichSchema, "missing">; // => Rich
+ * ```
+ */
+type LocalizedRich<RichSchema, K extends string> = RichSchema extends {
+    "{locale}": infer LM;
+} ? IsNever<AtPath<LM, K>> extends true ? Rich : AtPath<LM, K> extends MessageObject ? AtPath<LM, K> : Rich : Rich;
+/**
+ * Rich tag map resolved under a scoped prefix key.
+ *
+ * Internally composes the full dot-path (`"${PK}.${K}"`)
+ * and delegates resolution to `LocalizedRich`.
+ *
+ * @example
+ * ```ts
+ * type RichSchema = { "{locale}": { app: { link: { a: { href: string } } } } };
+ * ScopedRich<RichSchema, "app", "link">; // => { a: { href: string } }
+ * ScopedRich<RichSchema, "app", "missing">; // => Rich
+ * ```
+ */
+type ScopedRich<RichSchema, PK extends string | undefined, K extends string> = LocalizedRich<RichSchema, `${PK}.${K}`>;
+
 /**
  * Options for initializing a translator
  *
@@ -418,9 +458,6 @@ declare class ScopeTranslator<M extends LocaleMessages | unknown = unknown, Repl
     scoped<PK extends LocalizedPreKey<M> | undefined = undefined>(preKey?: PK): PK extends string ? ScopeTranslatorMethods<M, ReplacementSchema, PK> : ScopeTranslatorMethods<M, ReplacementSchema>;
 }
 
-/** Semantic tag attributes map. */
-type Attributes = Record<string, string>;
-
 /** Semantic node produced by the AST builder. */
 type ASTNode = TextNode | TagNode | RawNode;
 /** Plain text node in the semantic AST. */
@@ -441,6 +478,46 @@ interface RawNode {
     value: Exclude<MessageValue, string>;
 }
 
+/** Flat semantic token produced by the message tokenizer. */
+type Token = TextToken | TagOpenToken | TagCloseToken;
+/** Plain text segment in the message. */
+interface TextToken {
+    type: "text";
+    value: string;
+    position: number;
+}
+/** Opening semantic tag with parsed attributes. */
+interface TagOpenToken {
+    type: "tag-open";
+    name: string;
+    attributes: Attributes;
+    position: number;
+}
+/** Closing semantic tag (no attributes). */
+interface TagCloseToken {
+    type: "tag-close";
+    name: string;
+    position: number;
+}
+
+/**
+ * Tokenize a localized message string into semantic tokens.
+ *
+ * Supported syntax:
+ *   - <tag>text</tag>
+ *   - <tag key="value">text</tag>
+ *
+ * Notes:
+ * - Produces a flat token stream (nesting is handled in a later stage)
+ * - Tag names are identifier-like and case-sensitive
+ * - Attributes are strict key="value" pairs (double quotes only)
+ * - Variables are assumed to be interpolated beforehand
+ *
+ * This tokenizer is intentionally minimal and fail-closed:
+ * any unrecognized or partially valid syntax is treated as plain text.
+ */
+declare const tokenize: (message: string) => Token[];
+
 /**
  * Parse a rich message value into a semantic AST.
  *
@@ -490,4 +567,4 @@ interface Renderer<Output> {
  */
 declare function renderRichMessage<Output>(message: MessageValue, renderer: Renderer<Output>): Output[];
 
-export { type ASTNode, type AtPath, type Attributes, type FallbackLocalesMap, type FormatHandler, type GeneratePaths, type HandlerContext, type IfLocaleMessages, type IfMessageObject, type Key, type LoadingHandler, type Locale, type LocaleMessages, type LocalizedKey, type LocalizedPreKey, type LocalizedReplacement, type LocalizedValue, type MessageObject, type MessageValue, type MissingHandler, type PreKey, type Renderer, type Replacement, type ScopedKey, type ScopedReplacement, type ScopedValue, type TranslateConfig, type TranslateContext, type TranslateHandlers, type TranslateHook, ScopeTranslator as Translator, type ScopeTranslatorMethods as TranslatorMethods, type ScopeTranslatorOptions as TranslatorOptions, type TranslatorPlugin, type Value, parseRichMessage, renderRichMessage };
+export { type ASTNode, type AtPath, type Attributes, type FallbackLocalesMap, type FormatHandler, type GeneratePaths, type HandlerContext, type IfLocaleMessages, type IfMessageObject, type Key, type LoadingHandler, type Locale, type LocaleMessages, type LocalizedKey, type LocalizedPreKey, type LocalizedReplacement, type LocalizedRich, type LocalizedValue, type MessageObject, type MessageValue, type MissingHandler, type PreKey, type Renderer, type Replacement, type Rich, type ScopedKey, type ScopedReplacement, type ScopedRich, type ScopedValue, type Token, type TranslateConfig, type TranslateContext, type TranslateHandlers, type TranslateHook, ScopeTranslator as Translator, type ScopeTranslatorMethods as TranslatorMethods, type ScopeTranslatorOptions as TranslatorOptions, type TranslatorPlugin, type Value, parseRichMessage, renderRichMessage, tokenize };

package/dist/index.js

@@ -515,4 +515,4 @@ function renderRichMessage(message, renderer) {
   return render(nodes, renderer);
 }
 
-export { ScopeTranslator as Translator, parseRichMessage, renderRichMessage };
+export { ScopeTranslator as Translator, parseRichMessage, renderRichMessage, tokenize };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "intor-translator",
-  "version": "1.4.9",
+  "version": "1.4.11",
   "description": "🤖 A modern, type-safe i18n engine.",
   "author": {
     "name": "Yiming Liao",