@formatjs/intl-segmenter 11.5.0 → 11.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/polyfill-force.js +4 -2
- package/lib/polyfill.js +6 -4
- package/lib/should-polyfill.js +5 -1
- package/lib/src/cldr-segmentation-rules.generated.js +4 -1
- package/lib/src/segmentation-utils.js +7 -2
- package/lib/src/segmenter.js +32 -29
- package/lib/test262-main.js +3 -1
- package/package.json +3 -3
- package/polyfill.iife.js +9160 -1874
- package/src/segmenter.js +7 -7
package/lib/polyfill-force.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
var segmenter_1 = require("./src/segmenter");
|
|
2
4
|
Object.defineProperty(Intl, 'Segmenter', {
|
|
3
|
-
value: Segmenter,
|
|
5
|
+
value: segmenter_1.Segmenter,
|
|
4
6
|
enumerable: false,
|
|
5
7
|
writable: true,
|
|
6
8
|
configurable: true,
|
package/lib/polyfill.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
var segmenter_1 = require("./src/segmenter");
|
|
4
|
+
var should_polyfill_1 = require("./should-polyfill");
|
|
5
|
+
if ((0, should_polyfill_1.shouldPolyfill)()) {
|
|
4
6
|
Object.defineProperty(Intl, 'Segmenter', {
|
|
5
|
-
value: Segmenter,
|
|
7
|
+
value: segmenter_1.Segmenter,
|
|
6
8
|
enumerable: false,
|
|
7
9
|
writable: true,
|
|
8
10
|
configurable: true,
|
package/lib/should-polyfill.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SegmentationRules = void 0;
|
|
1
4
|
/* @generated */
|
|
2
5
|
// prettier-ignore
|
|
3
|
-
|
|
6
|
+
exports.SegmentationRules = {
|
|
4
7
|
"de": {
|
|
5
8
|
"sentence": {
|
|
6
9
|
"segmentRules": {},
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.isSurrogate = exports.replaceVariables = void 0;
|
|
4
|
+
var replaceVariables = function (variables, input) {
|
|
2
5
|
var findVarRegex = /\$[A-Za-z0-9_]+/gm;
|
|
3
6
|
return input.replaceAll(findVarRegex, function (match) {
|
|
4
7
|
if (!(match in variables)) {
|
|
@@ -7,12 +10,14 @@ export var replaceVariables = function (variables, input) {
|
|
|
7
10
|
return variables[match];
|
|
8
11
|
});
|
|
9
12
|
};
|
|
10
|
-
|
|
13
|
+
exports.replaceVariables = replaceVariables;
|
|
14
|
+
var isSurrogate = function (str, pos) {
|
|
11
15
|
return (0xd800 <= str.charCodeAt(pos - 1) &&
|
|
12
16
|
str.charCodeAt(pos - 1) <= 0xdbff &&
|
|
13
17
|
0xdc00 <= str.charCodeAt(pos) &&
|
|
14
18
|
str.charCodeAt(pos) <= 0xdfff);
|
|
15
19
|
};
|
|
20
|
+
exports.isSurrogate = isSurrogate;
|
|
16
21
|
// alternative surrogate check mimicking the java implementation
|
|
17
22
|
// const TRAIL_SURROGATE_BITMASK = 0xfffffc00
|
|
18
23
|
// const TRAIL_SURROGATE_BITS = 0xdc00
|
package/lib/src/segmenter.js
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Segmenter = void 0;
|
|
4
|
+
var tslib_1 = require("tslib");
|
|
5
|
+
var cldr_segmentation_rules_generated_1 = require("./cldr-segmentation-rules.generated");
|
|
6
|
+
var segmentation_utils_1 = require("./segmentation-utils");
|
|
7
|
+
var ecma402_abstract_1 = require("@formatjs/ecma402-abstract");
|
|
8
|
+
var intl_localematcher_1 = require("@formatjs/intl-localematcher");
|
|
6
9
|
/**
|
|
7
10
|
* Adds $ to before rules and ^ to after rules for strickness
|
|
8
11
|
* Replaces variables
|
|
@@ -14,7 +17,7 @@ import { ResolveLocale } from '@formatjs/intl-localematcher';
|
|
|
14
17
|
* @returns
|
|
15
18
|
*/
|
|
16
19
|
var generateRuleRegex = function (rule, variables, after) {
|
|
17
|
-
return new RegExp("".concat(after ? '^' : '').concat(replaceVariables(variables, rule)).concat(after ? '' : '$'));
|
|
20
|
+
return new RegExp("".concat(after ? '^' : '').concat((0, segmentation_utils_1.replaceVariables)(variables, rule)).concat(after ? '' : '$'));
|
|
18
21
|
};
|
|
19
22
|
var prepareLocaleSegmentationRules = function (segmentationTypeValue) {
|
|
20
23
|
var preparedRules = {};
|
|
@@ -38,35 +41,35 @@ var breaksAtResult = function (breaks, matchingRule) { return ({
|
|
|
38
41
|
breaks: breaks,
|
|
39
42
|
matchingRule: matchingRule,
|
|
40
43
|
}); };
|
|
41
|
-
|
|
44
|
+
var Segmenter = /** @class */ (function () {
|
|
42
45
|
function Segmenter(locales, options) {
|
|
43
46
|
var _newTarget = this.constructor;
|
|
44
47
|
if (_newTarget === undefined) {
|
|
45
48
|
throw TypeError("Constructor Intl.Segmenter requires 'new'");
|
|
46
49
|
}
|
|
47
|
-
var requestedLocales = CanonicalizeLocaleList(locales);
|
|
48
|
-
options = GetOptionsObject(options);
|
|
50
|
+
var requestedLocales = (0, ecma402_abstract_1.CanonicalizeLocaleList)(locales);
|
|
51
|
+
options = (0, ecma402_abstract_1.GetOptionsObject)(options);
|
|
49
52
|
var opt = Object.create(null);
|
|
50
|
-
var matcher = GetOption(options, 'localeMatcher', 'string', ['lookup', 'best fit'], 'best fit');
|
|
53
|
+
var matcher = (0, ecma402_abstract_1.GetOption)(options, 'localeMatcher', 'string', ['lookup', 'best fit'], 'best fit');
|
|
51
54
|
opt.localeMatcher = matcher;
|
|
52
|
-
var granularity = GetOption(options, 'granularity', 'string', ['word', 'sentence', 'grapheme'], 'grapheme');
|
|
55
|
+
var granularity = (0, ecma402_abstract_1.GetOption)(options, 'granularity', 'string', ['word', 'sentence', 'grapheme'], 'grapheme');
|
|
53
56
|
setSlot(this, 'granularity', granularity);
|
|
54
57
|
//TODO: figure out correct availible locales
|
|
55
|
-
var r = ResolveLocale(Segmenter.availableLocales, //availible locales
|
|
58
|
+
var r = (0, intl_localematcher_1.ResolveLocale)(Segmenter.availableLocales, //availible locales
|
|
56
59
|
requestedLocales, opt, [], // there is no relevantExtensionKeys
|
|
57
60
|
{}, function () { return ''; } //use only root rules
|
|
58
61
|
);
|
|
59
62
|
setSlot(this, 'locale', r.locale);
|
|
60
63
|
//root rules based on granularity
|
|
61
|
-
this.mergedSegmentationTypeValue = SegmentationRules.root[granularity];
|
|
64
|
+
this.mergedSegmentationTypeValue = cldr_segmentation_rules_generated_1.SegmentationRules.root[granularity];
|
|
62
65
|
//merge root rules with locale ones if locale is specified
|
|
63
66
|
if (r.locale.length) {
|
|
64
|
-
var localeOverrides = SegmentationRules[r.locale];
|
|
67
|
+
var localeOverrides = cldr_segmentation_rules_generated_1.SegmentationRules[r.locale];
|
|
65
68
|
if (granularity in localeOverrides) {
|
|
66
69
|
var localeSegmentationTypeValue = localeOverrides[granularity];
|
|
67
|
-
this.mergedSegmentationTypeValue.variables = __assign(__assign({}, this.mergedSegmentationTypeValue.variables), localeSegmentationTypeValue.variables);
|
|
68
|
-
this.mergedSegmentationTypeValue.segmentRules = __assign(__assign({}, this.mergedSegmentationTypeValue.segmentRules), localeSegmentationTypeValue.segmentRules);
|
|
69
|
-
this.mergedSegmentationTypeValue.suppressions = __spreadArray(__spreadArray([], this.mergedSegmentationTypeValue.suppressions, true), localeSegmentationTypeValue.suppressions, true);
|
|
70
|
+
this.mergedSegmentationTypeValue.variables = tslib_1.__assign(tslib_1.__assign({}, this.mergedSegmentationTypeValue.variables), localeSegmentationTypeValue.variables);
|
|
71
|
+
this.mergedSegmentationTypeValue.segmentRules = tslib_1.__assign(tslib_1.__assign({}, this.mergedSegmentationTypeValue.segmentRules), localeSegmentationTypeValue.segmentRules);
|
|
72
|
+
this.mergedSegmentationTypeValue.suppressions = tslib_1.__spreadArray(tslib_1.__spreadArray([], this.mergedSegmentationTypeValue.suppressions, true), localeSegmentationTypeValue.suppressions, true);
|
|
70
73
|
}
|
|
71
74
|
}
|
|
72
75
|
//prepare rules
|
|
@@ -75,7 +78,6 @@ export var Segmenter = /** @class */ (function () {
|
|
|
75
78
|
this.ruleSortedKeys = Object.keys(this.rules).sort(function (a, b) { return Number(a) - Number(b); });
|
|
76
79
|
}
|
|
77
80
|
Segmenter.prototype.breaksAt = function (position, input) {
|
|
78
|
-
var _a;
|
|
79
81
|
var ruleSortedKeys = this.ruleSortedKeys;
|
|
80
82
|
var rules = this.rules;
|
|
81
83
|
var mergedSegmentationTypeValue = this.mergedSegmentationTypeValue;
|
|
@@ -89,24 +91,24 @@ export var Segmenter = /** @class */ (function () {
|
|
|
89
91
|
}
|
|
90
92
|
//artificial rule 0.1: js specific, due to es5 regex not being unicode aware
|
|
91
93
|
//number 0.1 chosen to mimic java implementation, but needs to execute after 0.2 and 0.3 to be inside the string bounds
|
|
92
|
-
if (isSurrogate(input, position)) {
|
|
94
|
+
if ((0, segmentation_utils_1.isSurrogate)(input, position)) {
|
|
93
95
|
return breaksAtResult(false, '0.1');
|
|
94
96
|
}
|
|
95
97
|
var stringBeforeBreak = input.substring(0, position);
|
|
96
98
|
var stringAfterBreak = input.substring(position);
|
|
97
99
|
//artificial rule 0.4: handle suppressions
|
|
98
100
|
if ('suppressions' in mergedSegmentationTypeValue) {
|
|
99
|
-
for (var _i = 0,
|
|
100
|
-
var suppressions =
|
|
101
|
+
for (var _i = 0, _a = mergedSegmentationTypeValue.suppressions; _i < _a.length; _i++) {
|
|
102
|
+
var suppressions = _a[_i];
|
|
101
103
|
if (stringBeforeBreak.trim().endsWith(suppressions)) {
|
|
102
104
|
return breaksAtResult(false, '0.4');
|
|
103
105
|
}
|
|
104
106
|
}
|
|
105
107
|
}
|
|
106
108
|
// loop through rules and find a match
|
|
107
|
-
for (var
|
|
108
|
-
var ruleKey = ruleSortedKeys_1[
|
|
109
|
-
var
|
|
109
|
+
for (var _b = 0, ruleSortedKeys_1 = ruleSortedKeys; _b < ruleSortedKeys_1.length; _b++) {
|
|
110
|
+
var ruleKey = ruleSortedKeys_1[_b];
|
|
111
|
+
var _c = rules[ruleKey], before = _c.before, after = _c.after, breaks = _c.breaks;
|
|
110
112
|
// for debugging
|
|
111
113
|
// if (ruleKey === '16' && position === 4) {
|
|
112
114
|
// console.log({before, after, stringBeforeBreak, stringAfterBreak})
|
|
@@ -134,15 +136,16 @@ export var Segmenter = /** @class */ (function () {
|
|
|
134
136
|
};
|
|
135
137
|
Segmenter.prototype.resolvedOptions = function () {
|
|
136
138
|
checkReceiver(this, 'resolvedOptions');
|
|
137
|
-
return __assign({}, getMultiInternalSlots(__INTERNAL_SLOT_MAP__, this, 'locale', 'granularity'));
|
|
139
|
+
return tslib_1.__assign({}, (0, ecma402_abstract_1.getMultiInternalSlots)(__INTERNAL_SLOT_MAP__, this, 'locale', 'granularity'));
|
|
138
140
|
};
|
|
139
141
|
Segmenter.supportedLocalesOf = function (locales, options) {
|
|
140
|
-
return SupportedLocales(Segmenter.availableLocales, CanonicalizeLocaleList(locales), options);
|
|
142
|
+
return (0, ecma402_abstract_1.SupportedLocales)(Segmenter.availableLocales, (0, ecma402_abstract_1.CanonicalizeLocaleList)(locales), options);
|
|
141
143
|
};
|
|
142
|
-
Segmenter.availableLocales = new Set(Object.keys(SegmentationRules).filter(function (key) { return key !== 'root'; }));
|
|
144
|
+
Segmenter.availableLocales = new Set(Object.keys(cldr_segmentation_rules_generated_1.SegmentationRules).filter(function (key) { return key !== 'root'; }));
|
|
143
145
|
Segmenter.polyfilled = true;
|
|
144
146
|
return Segmenter;
|
|
145
147
|
}());
|
|
148
|
+
exports.Segmenter = Segmenter;
|
|
146
149
|
var createSegmentDataObject = function (segmenter, segment, index, input, matchingRule) {
|
|
147
150
|
var returnValue = {
|
|
148
151
|
segment: segment,
|
|
@@ -230,10 +233,10 @@ var SegmentIterator = /** @class */ (function () {
|
|
|
230
233
|
}());
|
|
231
234
|
var __INTERNAL_SLOT_MAP__ = new WeakMap();
|
|
232
235
|
function getSlot(instance, key) {
|
|
233
|
-
return getInternalSlot(__INTERNAL_SLOT_MAP__, instance, key);
|
|
236
|
+
return (0, ecma402_abstract_1.getInternalSlot)(__INTERNAL_SLOT_MAP__, instance, key);
|
|
234
237
|
}
|
|
235
238
|
function setSlot(instance, key, value) {
|
|
236
|
-
setInternalSlot(__INTERNAL_SLOT_MAP__, instance, key, value);
|
|
239
|
+
(0, ecma402_abstract_1.setInternalSlot)(__INTERNAL_SLOT_MAP__, instance, key, value);
|
|
237
240
|
}
|
|
238
241
|
function checkReceiver(receiver, methodName) {
|
|
239
242
|
if (!(receiver instanceof Segmenter)) {
|
package/lib/test262-main.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@formatjs/intl-segmenter",
|
|
3
|
-
"version": "11.5.
|
|
3
|
+
"version": "11.5.2",
|
|
4
4
|
"description": "Polyfill for Intl.Segmenter",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"intl",
|
|
@@ -20,8 +20,8 @@
|
|
|
20
20
|
},
|
|
21
21
|
"dependencies": {
|
|
22
22
|
"tslib": "^2.4.0",
|
|
23
|
-
"@formatjs/
|
|
24
|
-
"@formatjs/
|
|
23
|
+
"@formatjs/intl-localematcher": "0.5.1",
|
|
24
|
+
"@formatjs/ecma402-abstract": "1.17.4"
|
|
25
25
|
},
|
|
26
26
|
"main": "index.js",
|
|
27
27
|
"types": "index.d.ts",
|