@formatjs/intl-segmenter 11.2.0

package/src/segmentation-utils.d.ts

@@ -0,0 +1,3 @@
+export declare const replaceVariables: (variables: Record<string, string>, input: string) => string;
+export declare const isSurrogate: (str: string, pos: number) => boolean;
+//# sourceMappingURL=segmentation-utils.d.ts.map

package/src/segmentation-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"segmentation-utils.d.ts","sourceRoot":"","sources":["segmentation-utils.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,gBAAgB,cAChB,OAAO,MAAM,EAAE,MAAM,CAAC,SAC1B,MAAM,WASd,CAAA;AAED,eAAO,MAAM,WAAW,QAAS,MAAM,OAAO,MAAM,YAOnD,CAAA"}

package/src/segmentation-utils.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isSurrogate = exports.replaceVariables = void 0;
+var replaceVariables = function (variables, input) {
+    var findVarRegex = /\$[A-Za-z0-9_]+/gm;
+    return input.replaceAll(findVarRegex, function (match) {
+        if (!(match in variables)) {
+            throw new Error("No such variable ".concat(match));
+        }
+        return variables[match];
+    });
+};
+exports.replaceVariables = replaceVariables;
+var isSurrogate = function (str, pos) {
+    return (0xd800 <= str.charCodeAt(pos - 1) &&
+        str.charCodeAt(pos - 1) <= 0xdbff &&
+        0xdc00 <= str.charCodeAt(pos) &&
+        str.charCodeAt(pos) <= 0xdfff);
+};
+exports.isSurrogate = isSurrogate;
+// alternative surrogate check mimicking the java implementation
+// const TRAIL_SURROGATE_BITMASK = 0xfffffc00
+// const TRAIL_SURROGATE_BITS = 0xdc00
+// const LEAD_SURROGATE_BITMASK = 0xfffffc00
+// const LEAD_SURROGATE_BITS = 0xd800
+// const isSurrogate = (text: string, position: number) => {
+//   if (
+//     (text.charCodeAt(position - 1) & LEAD_SURROGATE_BITMASK) ==
+//       LEAD_SURROGATE_BITS &&
+//     (text.charCodeAt(position) & TRAIL_SURROGATE_BITMASK) ==
+//       TRAIL_SURROGATE_BITS
+//   ) {
+//     return true
+//   } else {
+//     return false
+//   }
+// }

package/src/segmenter.d.ts

@@ -0,0 +1,56 @@
+type SegmentResult = {
+    segment: string;
+    breakingRule?: string;
+    nonBreakingRules?: string[];
+} | undefined;
+export interface SegmenterOptions {
+    localeMatcher?: 'lookup' | 'best fit';
+    granularity?: 'word' | 'sentence' | 'grapheme';
+}
+export interface SegmenterResolvedOptions {
+    locale: string;
+    granularity: NonNullable<SegmenterOptions['granularity']>;
+}
+declare const breaksAtResult: (breaks: boolean, matchingRule: string) => {
+    breaks: boolean;
+    matchingRule: string;
+};
+export declare class Segmenter {
+    private readonly rules;
+    private readonly ruleSortedKeys;
+    private readonly mergedSegmentationTypeValue;
+    constructor(locales: string | string[] | undefined, options: SegmenterOptions);
+    breaksAt(position: number, input: string): ReturnType<typeof breaksAtResult>;
+    segment(input: string): SegmentIterator;
+    resolvedOptions(): SegmenterResolvedOptions;
+    static availableLocales: Set<string>;
+    static supportedLocalesOf(locales?: string | string[], options?: Pick<SegmenterOptions, 'localeMatcher'>): string[];
+    static readonly polyfilled = true;
+}
+declare class SegmentIterator implements Iterable<SegmentResult>, Iterator<SegmentResult> {
+    private readonly segmenter;
+    private lastSegmentIndex;
+    private input;
+    constructor(segmenter: Segmenter, input: string);
+    [Symbol.iterator](): SegmentIterator;
+    next(): {
+        done: boolean;
+        value: {
+            segment: string;
+            index: number;
+            input: string;
+            isWordLike?: boolean | undefined;
+        };
+    } | {
+        done: boolean;
+        value: undefined;
+    };
+    containing(positionInput: number): {
+        segment: string;
+        index: number;
+        input: string;
+        isWordLike?: boolean | undefined;
+    } | undefined;
+}
+export type { SegmentIterator };
+//# sourceMappingURL=segmenter.d.ts.map

package/src/segmenter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"segmenter.d.ts","sourceRoot":"","sources":["segmenter.ts"],"names":[],"mappings":"AA+BA,KAAK,aAAa,GACd;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAA;CAAC,GACrE,SAAS,CAAA;AAEb,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,EAAE,QAAQ,GAAG,UAAU,CAAA;IACrC,WAAW,CAAC,EAAE,MAAM,GAAG,UAAU,GAAG,UAAU,CAAA;CAC/C;AAED,MAAM,WAAW,wBAAwB;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,WAAW,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC,CAAA;CAC1D;AAoDD,QAAA,MAAM,cAAc,WAAY,OAAO,gBAAgB,MAAM;;;CAG3D,CAAA;AAEF,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAA;IACtB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAA;IAC/B,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAyB;gBAGnE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,EACtC,OAAO,EAAE,gBAAgB;IA0EpB,QAAQ,CACb,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GACZ,UAAU,CAAC,OAAO,cAAc,CAAC;IA6DpC,OAAO,CAAC,KAAK,EAAE,MAAM;IAKrB,eAAe,IAAI,wBAAwB;IAY3C,MAAM,CAAC,gBAAgB,cAEtB;IACD,MAAM,CAAC,kBAAkB,CACvB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,EAC3B,OAAO,CAAC,EAAE,IAAI,CAAC,gBAAgB,EAAE,eAAe,CAAC,GAChD,MAAM,EAAE;IAOX,gBAAuB,UAAU,QAAO;CACzC;AAwBD,cAAM,eACJ,YAAW,QAAQ,CAAC,aAAa,CAAC,EAAE,QAAQ,CAAC,aAAa,CAAC;IAE3D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAA;IAC1B,OAAO,CAAC,gBAAgB,CAAA;IACxB,OAAO,CAAC,KAAK,CAAA;gBACD,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM;IAS/C,CAAC,MAAM,CAAC,QAAQ,CAAC;IAIjB,IAAI;;;;;;;;;;;;IA+BJ,UAAU,CAAC,aAAa,EAAE,MAAM;;;;;;CAqDjC;AAED,YAAY,EAAC,eAAe,EAAC,CAAA"}

package/src/segmenter.js

@@ -0,0 +1,273 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Segmenter = void 0;
+var tslib_1 = require("tslib");
+var cldr_segmentation_rules_generated_1 = require("./cldr-segmentation-rules.generated");
+var segmentation_utils_1 = require("./segmentation-utils");
+var ecma402_abstract_1 = require("@formatjs/ecma402-abstract");
+var intl_localematcher_1 = require("@formatjs/intl-localematcher");
+/**
+ * Adds $ to before rules and ^ to after rules for strickness
+ * Replaces variables
+ * Initializes the RegExp
+ *
+ * @param rule raw rule string from cldr-segmentation-rules.generated
+ * @param variables
+ * @param after appends ^ if true and $ if false
+ * @returns
+ */
+var generateRuleRegex = function (rule, variables, after) {
+    return new RegExp("".concat(after ? '^' : '').concat((0, segmentation_utils_1.replaceVariables)(variables, rule)).concat(after ? '' : '$'));
+};
+var prepareLocaleSegmentationRules = function (segmentationTypeValue) {
+    var preparedRules = {};
+    for (var _i = 0, _a = Object.keys(segmentationTypeValue.segmentRules); _i < _a.length; _i++) {
+        var ruleNr = _a[_i];
+        var ruleValue = segmentationTypeValue.segmentRules[ruleNr];
+        var preparedRule = {
+            breaks: ruleValue.breaks,
+        };
+        if ('before' in ruleValue && ruleValue.before) {
+            preparedRule.before = generateRuleRegex(ruleValue.before, segmentationTypeValue.variables, false);
+        }
+        if ('after' in ruleValue && ruleValue.after) {
+            preparedRule.after = generateRuleRegex(ruleValue.after, segmentationTypeValue.variables, true);
+        }
+        preparedRules[ruleNr] = preparedRule;
+    }
+    return preparedRules;
+};
+var breaksAtResult = function (breaks, matchingRule) { return ({
+    breaks: breaks,
+    matchingRule: matchingRule,
+}); };
+var Segmenter = /** @class */ (function () {
+    function Segmenter(locales, options) {
+        var _newTarget = this.constructor;
+        if (_newTarget === undefined) {
+            throw TypeError("Constructor Intl.Segmenter requires 'new'");
+        }
+        var requestedLocales = (0, ecma402_abstract_1.CanonicalizeLocaleList)(locales);
+        options = (0, ecma402_abstract_1.GetOptionsObject)(options);
+        var opt = Object.create(null);
+        var matcher = (0, ecma402_abstract_1.GetOption)(options, 'localeMatcher', 'string', ['lookup', 'best fit'], 'best fit');
+        opt.localeMatcher = matcher;
+        var granularity = (0, ecma402_abstract_1.GetOption)(options, 'granularity', 'string', ['word', 'sentence', 'grapheme'], 'grapheme');
+        setSlot(this, 'granularity', granularity);
+        //TODO: figure out correct availible locales
+        var r = (0, intl_localematcher_1.ResolveLocale)(Segmenter.availableLocales, //availible locales
+        requestedLocales, opt, [], // there is no relevantExtensionKeys
+        {}, function () { return ''; } //use only root rules
+        );
+        setSlot(this, 'locale', r.locale);
+        //root rules based on granularity
+        this.mergedSegmentationTypeValue = cldr_segmentation_rules_generated_1.SegmentationRules.root[granularity];
+        //merge root rules with locale ones if locale is specified
+        if (r.locale.length) {
+            var localeOverrides = cldr_segmentation_rules_generated_1.SegmentationRules[r.locale];
+            if (granularity in localeOverrides) {
+                var localeSegmentationTypeValue = localeOverrides[granularity];
+                this.mergedSegmentationTypeValue.variables = tslib_1.__assign(tslib_1.__assign({}, this.mergedSegmentationTypeValue.variables), localeSegmentationTypeValue.variables);
+                this.mergedSegmentationTypeValue.segmentRules = tslib_1.__assign(tslib_1.__assign({}, this.mergedSegmentationTypeValue.segmentRules), localeSegmentationTypeValue.segmentRules);
+                this.mergedSegmentationTypeValue.suppressions = tslib_1.__spreadArray(tslib_1.__spreadArray([], this.mergedSegmentationTypeValue.suppressions, true), localeSegmentationTypeValue.suppressions, true);
+            }
+        }
+        //prepare rules
+        this.rules = prepareLocaleSegmentationRules(this.mergedSegmentationTypeValue);
+        //order rule keys
+        this.ruleSortedKeys = Object.keys(this.rules).sort(function (a, b) { return Number(a) - Number(b); });
+    }
+    Segmenter.prototype.breaksAt = function (position, input) {
+        var ruleSortedKeys = this.ruleSortedKeys;
+        var rules = this.rules;
+        var mergedSegmentationTypeValue = this.mergedSegmentationTypeValue;
+        //artificial rule 0.2
+        if (position === 0) {
+            return breaksAtResult(true, '0.2');
+        }
+        if (position === input.length) {
+            //rule 0.3
+            return breaksAtResult(true, '0.3');
+        }
+        //artificial rule 0.1: js specific, due to es5 regex not being unicode aware
+        //number 0.1 chosen to mimic java implementation, but needs to execute after 0.2 and 0.3 to be inside the string bounds
+        if ((0, segmentation_utils_1.isSurrogate)(input, position)) {
+            return breaksAtResult(false, '0.1');
+        }
+        var stringBeforeBreak = input.substring(0, position);
+        var stringAfterBreak = input.substring(position);
+        //artificial rule 0.4: handle suppressions
+        if ('suppressions' in mergedSegmentationTypeValue) {
+            for (var _i = 0, _a = mergedSegmentationTypeValue.suppressions; _i < _a.length; _i++) {
+                var suppressions = _a[_i];
+                if (stringBeforeBreak.trim().endsWith(suppressions)) {
+                    return breaksAtResult(false, '0.4');
+                }
+            }
+        }
+        // loop through rules and find a match
+        for (var _b = 0, ruleSortedKeys_1 = ruleSortedKeys; _b < ruleSortedKeys_1.length; _b++) {
+            var ruleKey = ruleSortedKeys_1[_b];
+            var _c = rules[ruleKey], before = _c.before, after = _c.after, breaks = _c.breaks;
+            // for debugging
+            // if (ruleKey === '16' && position === 4) {
+            //   console.log({before, after, stringBeforeBreak, stringAfterBreak})
+            // }
+            if (before) {
+                if (!before.test(stringBeforeBreak)) {
+                    //didn't match the before part, therfore skipping
+                    continue;
+                }
+            }
+            if (after) {
+                if (!after.test(stringAfterBreak)) {
+                    //didn't match the after part, therfore skipping
+                    continue;
+                }
+            }
+            return breaksAtResult(breaks, ruleKey);
+        }
+        //artificial rule 999: if no rule matched is Any ÷ Any so return true
+        return breaksAtResult(true, '999');
+    };
+    Segmenter.prototype.segment = function (input) {
+        checkReceiver(this, 'segment');
+        return new SegmentIterator(this, input);
+    };
+    Segmenter.prototype.resolvedOptions = function () {
+        checkReceiver(this, 'resolvedOptions');
+        return tslib_1.__assign({}, (0, ecma402_abstract_1.getMultiInternalSlots)(__INTERNAL_SLOT_MAP__, this, 'locale', 'granularity'));
+    };
+    Segmenter.supportedLocalesOf = function (locales, options) {
+        return (0, ecma402_abstract_1.SupportedLocales)(Segmenter.availableLocales, (0, ecma402_abstract_1.CanonicalizeLocaleList)(locales), options);
+    };
+    Segmenter.availableLocales = new Set(Object.keys(cldr_segmentation_rules_generated_1.SegmentationRules).filter(function (key) { return key !== 'root'; }));
+    Segmenter.polyfilled = true;
+    return Segmenter;
+}());
+exports.Segmenter = Segmenter;
+var createSegmentDataObject = function (segmenter, segment, index, input, matchingRule) {
+    var returnValue = {
+        segment: segment,
+        index: index,
+        input: input,
+    };
+    if (getSlot(segmenter, 'granularity') === 'word') {
+        returnValue.isWordLike = matchingRule !== '3.1' && matchingRule !== '3.2';
+    }
+    return returnValue;
+};
+var SegmentIterator = /** @class */ (function () {
+    function SegmentIterator(segmenter, input) {
+        this.segmenter = segmenter;
+        this.lastSegmentIndex = 0;
+        if (typeof input == 'symbol') {
+            throw TypeError("Input must not be a symbol");
+        }
+        this.input = String(input);
+    }
+    SegmentIterator.prototype[Symbol.iterator] = function () {
+        return new SegmentIterator(this.segmenter, this.input);
+    };
+    SegmentIterator.prototype.next = function () {
+        //using only the relevant bit of the string
+        var checkString = this.input.substring(this.lastSegmentIndex);
+        //loop from the start of the checkString, until exactly length (breaksAt returns break at pos=== lenght)
+        for (var position = 1; position <= checkString.length; position++) {
+            var _a = this.segmenter.breaksAt(position, checkString), breaks = _a.breaks, matchingRule = _a.matchingRule;
+            if (breaks) {
+                var segment = checkString.substring(0, position);
+                var index = this.lastSegmentIndex;
+                this.lastSegmentIndex += position;
+                return {
+                    done: false,
+                    value: createSegmentDataObject(this.segmenter, segment, index, this.input, matchingRule),
+                };
+            }
+        }
+        //no segment was found by the loop, therefore the segmentation is done
+        return { done: true, value: undefined };
+    };
+    SegmentIterator.prototype.containing = function (positionInput) {
+        if (typeof positionInput === 'bigint') {
+            throw TypeError('Index must not be a BigInt');
+        }
+        var position = Number(positionInput);
+        //https://tc39.es/ecma262/#sec-tointegerorinfinity
+        // 2. If number is NaN, +0𝔽, or -0𝔽, return 0.
+        if (isNaN(position) || !position) {
+            position = 0;
+        }
+        // 5. Let integer be floor(abs(ℝ(number))).
+        // 6. If number < -0𝔽, set integer to -integer.
+        position = Math.floor(Math.abs(position)) * (position < 0 ? -1 : 1);
+        if (position < 0 || position >= this.input.length) {
+            return undefined;
+        }
+        //find previous break point
+        var previousBreakPoint = 0;
+        if (position === 0) {
+            previousBreakPoint = 0;
+        }
+        else {
+            var checkString_1 = this.input;
+            for (var cursor = position; cursor >= 0; cursor--) {
+                var breaks = this.segmenter.breaksAt(cursor, checkString_1).breaks;
+                if (breaks) {
+                    previousBreakPoint = cursor;
+                    break;
+                }
+            }
+        }
+        var checkString = this.input.substring(previousBreakPoint);
+        //find next break point
+        for (var cursor = 1; cursor <= checkString.length; cursor++) {
+            var _a = this.segmenter.breaksAt(cursor, checkString), breaks = _a.breaks, matchingRule = _a.matchingRule;
+            if (breaks) {
+                var segment = checkString.substring(0, cursor);
+                return createSegmentDataObject(this.segmenter, segment, previousBreakPoint, this.input, matchingRule);
+            }
+        }
+    };
+    return SegmentIterator;
+}());
+var __INTERNAL_SLOT_MAP__ = new WeakMap();
+function getSlot(instance, key) {
+    return (0, ecma402_abstract_1.getInternalSlot)(__INTERNAL_SLOT_MAP__, instance, key);
+}
+function setSlot(instance, key, value) {
+    (0, ecma402_abstract_1.setInternalSlot)(__INTERNAL_SLOT_MAP__, instance, key, value);
+}
+function checkReceiver(receiver, methodName) {
+    if (!(receiver instanceof Segmenter)) {
+        throw TypeError("Method Intl.Segmenter.prototype.".concat(methodName, " called on incompatible receiver"));
+    }
+}
+try {
+    // IE11 does not have Symbol
+    if (typeof Symbol !== 'undefined') {
+        Object.defineProperty(Segmenter.prototype, Symbol.toStringTag, {
+            value: 'Intl.Segmenter',
+            writable: false,
+            enumerable: false,
+            configurable: true,
+        });
+    }
+    //github.com/tc39/test262/blob/main/test/intl402/Segmenter/constructor/length.js
+    https: Object.defineProperty(Segmenter.prototype.constructor, 'length', {
+        value: 0,
+        writable: false,
+        enumerable: false,
+        configurable: true,
+    });
+    // https://github.com/tc39/test262/blob/main/test/intl402/Segmenter/constructor/supportedLocalesOf/length.js
+    Object.defineProperty(Segmenter.supportedLocalesOf, 'length', {
+        value: 1,
+        writable: false,
+        enumerable: false,
+        configurable: true,
+    });
+}
+catch (e) {
+    // Meta fix so we're test262-compliant, not important
+}

package/test262-main.d.ts

@@ -0,0 +1,2 @@
+import './polyfill-force';
+//# sourceMappingURL=test262-main.d.ts.map

package/test262-main.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test262-main.d.ts","sourceRoot":"","sources":["test262-main.ts"],"names":[],"mappings":"AAEA,OAAO,kBAAkB,CAAC"}

package/test262-main.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+// @generated
+// @ts-nocheck
+require("./polyfill-force");