intor-translator 1.4.9 → 1.4.10

package/dist/index.cjs

@@ -520,3 +520,4 @@ function renderRichMessage(message, renderer) {
 exports.Translator = ScopeTranslator;
 exports.parseRichMessage = parseRichMessage;
 exports.renderRichMessage = renderRichMessage;
+exports.tokenize = tokenize;

package/dist/index.d.cts

@@ -441,6 +441,46 @@ interface RawNode {
     value: Exclude<MessageValue, string>;
 }
 
+/** Flat semantic token produced by the message tokenizer. */
+type Token = TextToken | TagOpenToken | TagCloseToken;
+/** Plain text segment in the message. */
+interface TextToken {
+    type: "text";
+    value: string;
+    position: number;
+}
+/** Opening semantic tag with parsed attributes. */
+interface TagOpenToken {
+    type: "tag-open";
+    name: string;
+    attributes: Attributes;
+    position: number;
+}
+/** Closing semantic tag (no attributes). */
+interface TagCloseToken {
+    type: "tag-close";
+    name: string;
+    position: number;
+}
+
+/**
+ * Tokenize a localized message string into semantic tokens.
+ *
+ * Supported syntax:
+ *   - <tag>text</tag>
+ *   - <tag key="value">text</tag>
+ *
+ * Notes:
+ * - Produces a flat token stream (nesting is handled in a later stage)
+ * - Tag names are identifier-like and case-sensitive
+ * - Attributes are strict key="value" pairs (double quotes only)
+ * - Variables are assumed to be interpolated beforehand
+ *
+ * This tokenizer is intentionally minimal and fail-closed:
+ * any unrecognized or partially valid syntax is treated as plain text.
+ */
+declare const tokenize: (message: string) => Token[];
+
 /**
  * Parse a rich message value into a semantic AST.
  *
@@ -490,4 +530,4 @@ interface Renderer<Output> {
  */
 declare function renderRichMessage<Output>(message: MessageValue, renderer: Renderer<Output>): Output[];
 
-export { type ASTNode, type AtPath, type Attributes, type FallbackLocalesMap, type FormatHandler, type GeneratePaths, type HandlerContext, type IfLocaleMessages, type IfMessageObject, type Key, type LoadingHandler, type Locale, type LocaleMessages, type LocalizedKey, type LocalizedPreKey, type LocalizedReplacement, type LocalizedValue, type MessageObject, type MessageValue, type MissingHandler, type PreKey, type Renderer, type Replacement, type ScopedKey, type ScopedReplacement, type ScopedValue, type TranslateConfig, type TranslateContext, type TranslateHandlers, type TranslateHook, ScopeTranslator as Translator, type ScopeTranslatorMethods as TranslatorMethods, type ScopeTranslatorOptions as TranslatorOptions, type TranslatorPlugin, type Value, parseRichMessage, renderRichMessage };
+export { type ASTNode, type AtPath, type Attributes, type FallbackLocalesMap, type FormatHandler, type GeneratePaths, type HandlerContext, type IfLocaleMessages, type IfMessageObject, type Key, type LoadingHandler, type Locale, type LocaleMessages, type LocalizedKey, type LocalizedPreKey, type LocalizedReplacement, type LocalizedValue, type MessageObject, type MessageValue, type MissingHandler, type PreKey, type Renderer, type Replacement, type ScopedKey, type ScopedReplacement, type ScopedValue, type Token, type TranslateConfig, type TranslateContext, type TranslateHandlers, type TranslateHook, ScopeTranslator as Translator, type ScopeTranslatorMethods as TranslatorMethods, type ScopeTranslatorOptions as TranslatorOptions, type TranslatorPlugin, type Value, parseRichMessage, renderRichMessage, tokenize };

package/dist/index.d.ts

@@ -441,6 +441,46 @@ interface RawNode {
     value: Exclude<MessageValue, string>;
 }
 
+/** Flat semantic token produced by the message tokenizer. */
+type Token = TextToken | TagOpenToken | TagCloseToken;
+/** Plain text segment in the message. */
+interface TextToken {
+    type: "text";
+    value: string;
+    position: number;
+}
+/** Opening semantic tag with parsed attributes. */
+interface TagOpenToken {
+    type: "tag-open";
+    name: string;
+    attributes: Attributes;
+    position: number;
+}
+/** Closing semantic tag (no attributes). */
+interface TagCloseToken {
+    type: "tag-close";
+    name: string;
+    position: number;
+}
+
+/**
+ * Tokenize a localized message string into semantic tokens.
+ *
+ * Supported syntax:
+ *   - <tag>text</tag>
+ *   - <tag key="value">text</tag>
+ *
+ * Notes:
+ * - Produces a flat token stream (nesting is handled in a later stage)
+ * - Tag names are identifier-like and case-sensitive
+ * - Attributes are strict key="value" pairs (double quotes only)
+ * - Variables are assumed to be interpolated beforehand
+ *
+ * This tokenizer is intentionally minimal and fail-closed:
+ * any unrecognized or partially valid syntax is treated as plain text.
+ */
+declare const tokenize: (message: string) => Token[];
+
 /**
  * Parse a rich message value into a semantic AST.
  *
@@ -490,4 +530,4 @@ interface Renderer<Output> {
  */
 declare function renderRichMessage<Output>(message: MessageValue, renderer: Renderer<Output>): Output[];
 
-export { type ASTNode, type AtPath, type Attributes, type FallbackLocalesMap, type FormatHandler, type GeneratePaths, type HandlerContext, type IfLocaleMessages, type IfMessageObject, type Key, type LoadingHandler, type Locale, type LocaleMessages, type LocalizedKey, type LocalizedPreKey, type LocalizedReplacement, type LocalizedValue, type MessageObject, type MessageValue, type MissingHandler, type PreKey, type Renderer, type Replacement, type ScopedKey, type ScopedReplacement, type ScopedValue, type TranslateConfig, type TranslateContext, type TranslateHandlers, type TranslateHook, ScopeTranslator as Translator, type ScopeTranslatorMethods as TranslatorMethods, type ScopeTranslatorOptions as TranslatorOptions, type TranslatorPlugin, type Value, parseRichMessage, renderRichMessage };
+export { type ASTNode, type AtPath, type Attributes, type FallbackLocalesMap, type FormatHandler, type GeneratePaths, type HandlerContext, type IfLocaleMessages, type IfMessageObject, type Key, type LoadingHandler, type Locale, type LocaleMessages, type LocalizedKey, type LocalizedPreKey, type LocalizedReplacement, type LocalizedValue, type MessageObject, type MessageValue, type MissingHandler, type PreKey, type Renderer, type Replacement, type ScopedKey, type ScopedReplacement, type ScopedValue, type Token, type TranslateConfig, type TranslateContext, type TranslateHandlers, type TranslateHook, ScopeTranslator as Translator, type ScopeTranslatorMethods as TranslatorMethods, type ScopeTranslatorOptions as TranslatorOptions, type TranslatorPlugin, type Value, parseRichMessage, renderRichMessage, tokenize };

package/dist/index.js

@@ -515,4 +515,4 @@ function renderRichMessage(message, renderer) {
   return render(nodes, renderer);
 }
 
-export { ScopeTranslator as Translator, parseRichMessage, renderRichMessage };
+export { ScopeTranslator as Translator, parseRichMessage, renderRichMessage, tokenize };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "intor-translator",
-  "version": "1.4.9",
+  "version": "1.4.10",
   "description": "🤖 A modern, type-safe i18n engine.",
   "author": {
     "name": "Yiming Liao",