gpt-tokenizer 2.0.0 → 2.1.0

package/src/__snapshots__/GptEncoding.test.ts.snap

@@ -0,0 +1,811 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`gpt-3.5-turbo chat functionality encodes a chat correctly 1`] = `
+[
+  100264,
+  9125,
+  198,
+  2675,
+  527,
+  264,
+  11190,
+  11,
+  5497,
+  93685,
+  287,
+  18328,
+  430,
+  48018,
+  13166,
+  503,
+  71921,
+  1139,
+  14733,
+  6498,
+  13,
+  100265,
+  198,
+  100264,
+  8858,
+  3398,
+  198,
+  3648,
+  80526,
+  552,
+  690,
+  1520,
+  6678,
+  1948,
+  8614,
+  6650,
+  13,
+  100265,
+  198,
+  100264,
+  8858,
+  12354,
+  11451,
+  198,
+  42575,
+  3318,
+  1664,
+  3871,
+  690,
+  5376,
+  13254,
+  13,
+  100265,
+  198,
+  100264,
+  8858,
+  3398,
+  198,
+  10267,
+  596,
+  12960,
+  1203,
+  994,
+  584,
+  617,
+  810,
+  34494,
+  311,
+  5916,
+  2385,
+  389,
+  10708,
+  369,
+  7319,
+  33164,
+  13,
+  100265,
+  198,
+  100264,
+  8858,
+  12354,
+  11451,
+  198,
+  10267,
+  596,
+  3137,
+  3010,
+  994,
+  584,
+  2351,
+  2753,
+  13326,
+  922,
+  1268,
+  311,
+  656,
+  2731,
+  13,
+  100265,
+  198,
+  100264,
+  882,
+  198,
+  2028,
+  3389,
+  27137,
+  3445,
+  584,
+  1541,
+  956,
+  617,
+  892,
+  311,
+  44790,
+  279,
+  18435,
+  369,
+  279,
+  3016,
+  6493,
+  481,
+  13,
+  100265,
+  198,
+  100264,
+  78191,
+  198,
+]
+`;
+
+exports[`gpt-3.5-turbo chat functionality encodes a chat correctly 2`] = `
+"<|im_start|>system
+You are a helpful, pattern-following assistant that translates corporate jargon into plain English.<|im_end|>
+<|im_start|>example_user
+New synergies will help drive top-line growth.<|im_end|>
+<|im_start|>example_assistant
+Things working well together will increase revenue.<|im_end|>
+<|im_start|>example_user
+Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.<|im_end|>
+<|im_start|>example_assistant
+Let's talk later when we're less busy about how to do better.<|im_end|>
+<|im_start|>user
+This late pivot means we don't have time to boil the ocean for the client deliverable.<|im_end|>
+<|im_start|>assistant
+"
+`;
+
+exports[`gpt-3.5-turbo-0301 chat functionality encodes a chat correctly 1`] = `
+[
+  100264,
+  9125,
+  198,
+  2675,
+  527,
+  264,
+  11190,
+  11,
+  5497,
+  93685,
+  287,
+  18328,
+  430,
+  48018,
+  13166,
+  503,
+  71921,
+  1139,
+  14733,
+  6498,
+  13,
+  100265,
+  198,
+  100264,
+  8858,
+  3398,
+  198,
+  3648,
+  80526,
+  552,
+  690,
+  1520,
+  6678,
+  1948,
+  8614,
+  6650,
+  13,
+  100265,
+  198,
+  100264,
+  8858,
+  12354,
+  11451,
+  198,
+  42575,
+  3318,
+  1664,
+  3871,
+  690,
+  5376,
+  13254,
+  13,
+  100265,
+  198,
+  100264,
+  8858,
+  3398,
+  198,
+  10267,
+  596,
+  12960,
+  1203,
+  994,
+  584,
+  617,
+  810,
+  34494,
+  311,
+  5916,
+  2385,
+  389,
+  10708,
+  369,
+  7319,
+  33164,
+  13,
+  100265,
+  198,
+  100264,
+  8858,
+  12354,
+  11451,
+  198,
+  10267,
+  596,
+  3137,
+  3010,
+  994,
+  584,
+  2351,
+  2753,
+  13326,
+  922,
+  1268,
+  311,
+  656,
+  2731,
+  13,
+  100265,
+  198,
+  100264,
+  882,
+  198,
+  2028,
+  3389,
+  27137,
+  3445,
+  584,
+  1541,
+  956,
+  617,
+  892,
+  311,
+  44790,
+  279,
+  18435,
+  369,
+  279,
+  3016,
+  6493,
+  481,
+  13,
+  100265,
+  198,
+  100264,
+  78191,
+  198,
+]
+`;
+
+exports[`gpt-3.5-turbo-0301 chat functionality encodes a chat correctly 2`] = `
+"<|im_start|>system
+You are a helpful, pattern-following assistant that translates corporate jargon into plain English.<|im_end|>
+<|im_start|>example_user
+New synergies will help drive top-line growth.<|im_end|>
+<|im_start|>example_assistant
+Things working well together will increase revenue.<|im_end|>
+<|im_start|>example_user
+Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.<|im_end|>
+<|im_start|>example_assistant
+Let's talk later when we're less busy about how to do better.<|im_end|>
+<|im_start|>user
+This late pivot means we don't have time to boil the ocean for the client deliverable.<|im_end|>
+<|im_start|>assistant
+"
+`;
+
+exports[`gpt-4 chat functionality encodes a chat correctly 1`] = `
+[
+  100264,
+  9125,
+  100266,
+  2675,
+  527,
+  264,
+  11190,
+  11,
+  5497,
+  93685,
+  287,
+  18328,
+  430,
+  48018,
+  13166,
+  503,
+  71921,
+  1139,
+  14733,
+  6498,
+  13,
+  100265,
+  100264,
+  8858,
+  3398,
+  100266,
+  3648,
+  80526,
+  552,
+  690,
+  1520,
+  6678,
+  1948,
+  8614,
+  6650,
+  13,
+  100265,
+  100264,
+  8858,
+  12354,
+  11451,
+  100266,
+  42575,
+  3318,
+  1664,
+  3871,
+  690,
+  5376,
+  13254,
+  13,
+  100265,
+  100264,
+  8858,
+  3398,
+  100266,
+  10267,
+  596,
+  12960,
+  1203,
+  994,
+  584,
+  617,
+  810,
+  34494,
+  311,
+  5916,
+  2385,
+  389,
+  10708,
+  369,
+  7319,
+  33164,
+  13,
+  100265,
+  100264,
+  8858,
+  12354,
+  11451,
+  100266,
+  10267,
+  596,
+  3137,
+  3010,
+  994,
+  584,
+  2351,
+  2753,
+  13326,
+  922,
+  1268,
+  311,
+  656,
+  2731,
+  13,
+  100265,
+  100264,
+  882,
+  100266,
+  2028,
+  3389,
+  27137,
+  3445,
+  584,
+  1541,
+  956,
+  617,
+  892,
+  311,
+  44790,
+  279,
+  18435,
+  369,
+  279,
+  3016,
+  6493,
+  481,
+  13,
+  100265,
+  100264,
+  78191,
+  100266,
+]
+`;
+
+exports[`gpt-4 chat functionality encodes a chat correctly 2`] = `"<|im_start|>system<|im_sep|>You are a helpful, pattern-following assistant that translates corporate jargon into plain English.<|im_end|><|im_start|>example_user<|im_sep|>New synergies will help drive top-line growth.<|im_end|><|im_start|>example_assistant<|im_sep|>Things working well together will increase revenue.<|im_end|><|im_start|>example_user<|im_sep|>Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.<|im_end|><|im_start|>example_assistant<|im_sep|>Let's talk later when we're less busy about how to do better.<|im_end|><|im_start|>user<|im_sep|>This late pivot means we don't have time to boil the ocean for the client deliverable.<|im_end|><|im_start|>assistant<|im_sep|>"`;
+
+exports[`gpt-4-0314 chat functionality encodes a chat correctly 1`] = `
+[
+  100264,
+  9125,
+  100266,
+  2675,
+  527,
+  264,
+  11190,
+  11,
+  5497,
+  93685,
+  287,
+  18328,
+  430,
+  48018,
+  13166,
+  503,
+  71921,
+  1139,
+  14733,
+  6498,
+  13,
+  100265,
+  100264,
+  8858,
+  3398,
+  100266,
+  3648,
+  80526,
+  552,
+  690,
+  1520,
+  6678,
+  1948,
+  8614,
+  6650,
+  13,
+  100265,
+  100264,
+  8858,
+  12354,
+  11451,
+  100266,
+  42575,
+  3318,
+  1664,
+  3871,
+  690,
+  5376,
+  13254,
+  13,
+  100265,
+  100264,
+  8858,
+  3398,
+  100266,
+  10267,
+  596,
+  12960,
+  1203,
+  994,
+  584,
+  617,
+  810,
+  34494,
+  311,
+  5916,
+  2385,
+  389,
+  10708,
+  369,
+  7319,
+  33164,
+  13,
+  100265,
+  100264,
+  8858,
+  12354,
+  11451,
+  100266,
+  10267,
+  596,
+  3137,
+  3010,
+  994,
+  584,
+  2351,
+  2753,
+  13326,
+  922,
+  1268,
+  311,
+  656,
+  2731,
+  13,
+  100265,
+  100264,
+  882,
+  100266,
+  2028,
+  3389,
+  27137,
+  3445,
+  584,
+  1541,
+  956,
+  617,
+  892,
+  311,
+  44790,
+  279,
+  18435,
+  369,
+  279,
+  3016,
+  6493,
+  481,
+  13,
+  100265,
+  100264,
+  78191,
+  100266,
+]
+`;
+
+exports[`gpt-4-0314 chat functionality encodes a chat correctly 2`] = `"<|im_start|>system<|im_sep|>You are a helpful, pattern-following assistant that translates corporate jargon into plain English.<|im_end|><|im_start|>example_user<|im_sep|>New synergies will help drive top-line growth.<|im_end|><|im_start|>example_assistant<|im_sep|>Things working well together will increase revenue.<|im_end|><|im_start|>example_user<|im_sep|>Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.<|im_end|><|im_start|>example_assistant<|im_sep|>Let's talk later when we're less busy about how to do better.<|im_end|><|im_start|>user<|im_sep|>This late pivot means we don't have time to boil the ocean for the client deliverable.<|im_end|><|im_start|>assistant<|im_sep|>"`;
+
+exports[`gpt-4-32k chat functionality encodes a chat correctly 1`] = `
+[
+  100264,
+  9125,
+  100266,
+  2675,
+  527,
+  264,
+  11190,
+  11,
+  5497,
+  93685,
+  287,
+  18328,
+  430,
+  48018,
+  13166,
+  503,
+  71921,
+  1139,
+  14733,
+  6498,
+  13,
+  100265,
+  100264,
+  8858,
+  3398,
+  100266,
+  3648,
+  80526,
+  552,
+  690,
+  1520,
+  6678,
+  1948,
+  8614,
+  6650,
+  13,
+  100265,
+  100264,
+  8858,
+  12354,
+  11451,
+  100266,
+  42575,
+  3318,
+  1664,
+  3871,
+  690,
+  5376,
+  13254,
+  13,
+  100265,
+  100264,
+  8858,
+  3398,
+  100266,
+  10267,
+  596,
+  12960,
+  1203,
+  994,
+  584,
+  617,
+  810,
+  34494,
+  311,
+  5916,
+  2385,
+  389,
+  10708,
+  369,
+  7319,
+  33164,
+  13,
+  100265,
+  100264,
+  8858,
+  12354,
+  11451,
+  100266,
+  10267,
+  596,
+  3137,
+  3010,
+  994,
+  584,
+  2351,
+  2753,
+  13326,
+  922,
+  1268,
+  311,
+  656,
+  2731,
+  13,
+  100265,
+  100264,
+  882,
+  100266,
+  2028,
+  3389,
+  27137,
+  3445,
+  584,
+  1541,
+  956,
+  617,
+  892,
+  311,
+  44790,
+  279,
+  18435,
+  369,
+  279,
+  3016,
+  6493,
+  481,
+  13,
+  100265,
+  100264,
+  78191,
+  100266,
+]
+`;
+
+exports[`gpt-4-32k chat functionality encodes a chat correctly 2`] = `"<|im_start|>system<|im_sep|>You are a helpful, pattern-following assistant that translates corporate jargon into plain English.<|im_end|><|im_start|>example_user<|im_sep|>New synergies will help drive top-line growth.<|im_end|><|im_start|>example_assistant<|im_sep|>Things working well together will increase revenue.<|im_end|><|im_start|>example_user<|im_sep|>Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.<|im_end|><|im_start|>example_assistant<|im_sep|>Let's talk later when we're less busy about how to do better.<|im_end|><|im_start|>user<|im_sep|>This late pivot means we don't have time to boil the ocean for the client deliverable.<|im_end|><|im_start|>assistant<|im_sep|>"`;
+
+exports[`gpt-4-32k-0314 chat functionality encodes a chat correctly 1`] = `
+[
+  100264,
+  9125,
+  100266,
+  2675,
+  527,
+  264,
+  11190,
+  11,
+  5497,
+  93685,
+  287,
+  18328,
+  430,
+  48018,
+  13166,
+  503,
+  71921,
+  1139,
+  14733,
+  6498,
+  13,
+  100265,
+  100264,
+  8858,
+  3398,
+  100266,
+  3648,
+  80526,
+  552,
+  690,
+  1520,
+  6678,
+  1948,
+  8614,
+  6650,
+  13,
+  100265,
+  100264,
+  8858,
+  12354,
+  11451,
+  100266,
+  42575,
+  3318,
+  1664,
+  3871,
+  690,
+  5376,
+  13254,
+  13,
+  100265,
+  100264,
+  8858,
+  3398,
+  100266,
+  10267,
+  596,
+  12960,
+  1203,
+  994,
+  584,
+  617,
+  810,
+  34494,
+  311,
+  5916,
+  2385,
+  389,
+  10708,
+  369,
+  7319,
+  33164,
+  13,
+  100265,
+  100264,
+  8858,
+  12354,
+  11451,
+  100266,
+  10267,
+  596,
+  3137,
+  3010,
+  994,
+  584,
+  2351,
+  2753,
+  13326,
+  922,
+  1268,
+  311,
+  656,
+  2731,
+  13,
+  100265,
+  100264,
+  882,
+  100266,
+  2028,
+  3389,
+  27137,
+  3445,
+  584,
+  1541,
+  956,
+  617,
+  892,
+  311,
+  44790,
+  279,
+  18435,
+  369,
+  279,
+  3016,
+  6493,
+  481,
+  13,
+  100265,
+  100264,
+  78191,
+  100266,
+]
+`;
+
+exports[`gpt-4-32k-0314 chat functionality encodes a chat correctly 2`] = `"<|im_start|>system<|im_sep|>You are a helpful, pattern-following assistant that translates corporate jargon into plain English.<|im_end|><|im_start|>example_user<|im_sep|>New synergies will help drive top-line growth.<|im_end|><|im_start|>example_assistant<|im_sep|>Things working well together will increase revenue.<|im_end|><|im_start|>example_user<|im_sep|>Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.<|im_end|><|im_start|>example_assistant<|im_sep|>Let's talk later when we're less busy about how to do better.<|im_end|><|im_start|>user<|im_sep|>This late pivot means we don't have time to boil the ocean for the client deliverable.<|im_end|><|im_start|>assistant<|im_sep|>"`;