079project 1.0.0

package/tokenizer.json

@@ -0,0 +1 @@
+{"class_name": "Tokenizer", "config": {"num_words": 5000, "filters": "", "lower": true, "split": " ", "char_level": false, "oov_token": "<UNK>", "document_count": 20, "word_counts": "{\"i\": 2, \"person\": 2, \"a\": 8, \"am\": 2, \"are\": 6, \"student\": 2, \"you\": 2, \"he\": 2, \"is\": 12, \"teacher\": 2, \"my\": 4, \"she\": 2, \"friend\": 2, \"this\": 2, \"book\": 2, \"dog\": 2, \"that\": 2, \"happy\": 2, \"we\": 2, \"they\": 2, \"here\": 2, \"it\": 2, \"raining\": 2, \"cat\": 2, \"the\": 2, \"black\": 2}", "word_docs": "{\"am\": 2, \"person\": 2, \"i\": 2, \"a\": 8, \"you\": 2, \"are\": 6, \"student\": 2, \"he\": 2, \"teacher\": 2, \"is\": 12, \"my\": 4, \"friend\": 2, \"she\": 2, \"this\": 2, \"book\": 2, \"dog\": 2, \"that\": 2, \"happy\": 2, \"we\": 2, \"they\": 2, \"here\": 2, \"raining\": 2, \"it\": 2, \"the\": 2, \"black\": 2, \"cat\": 2}", "index_docs": "{\"8\": 2, \"7\": 2, \"6\": 2, \"3\": 8, \"10\": 2, \"4\": 6, \"9\": 2, \"11\": 2, \"12\": 2, \"2\": 12, \"5\": 4, \"14\": 2, \"13\": 2, \"15\": 2, \"16\": 2, \"17\": 2, \"18\": 2, \"19\": 2, \"20\": 2, \"21\": 2, \"22\": 2, \"24\": 2, \"23\": 2, \"26\": 2, \"27\": 2, \"25\": 2}", "index_word": "{\"1\": \"<UNK>\", \"2\": \"is\", \"3\": \"a\", \"4\": \"are\", \"5\": \"my\", \"6\": \"i\", \"7\": \"person\", \"8\": \"am\", \"9\": \"student\", \"10\": \"you\", \"11\": \"he\", \"12\": \"teacher\", \"13\": \"she\", \"14\": \"friend\", \"15\": \"this\", \"16\": \"book\", \"17\": \"dog\", \"18\": \"that\", \"19\": \"happy\", \"20\": \"we\", \"21\": \"they\", \"22\": \"here\", \"23\": \"it\", \"24\": \"raining\", \"25\": \"cat\", \"26\": \"the\", \"27\": \"black\"}", "word_index": "{\"<UNK>\": 1, \"is\": 2, \"a\": 3, \"are\": 4, \"my\": 5, \"i\": 6, \"person\": 7, \"am\": 8, \"student\": 9, \"you\": 10, \"he\": 11, \"teacher\": 12, \"she\": 13, \"friend\": 14, \"this\": 15, \"book\": 16, \"dog\": 17, \"that\": 18, \"happy\": 19, \"we\": 20, \"they\": 21, \"here\": 22, \"it\": 23, \"raining\": 24, \"cat\": 25, \"the\": 26, \"black\": 27}"}}

package/vocabularySplitter.js

@@ -0,0 +1,253 @@
+const fs = require('fs');
+const path = require('path');
+const axios = require('axios');
+const readline = require('readline');
+const { createHash } = require('crypto');
+const cluster = require('cluster');
+const os = require('os');
+
+// 配置
+const VOCAB_FILES = [
+    './vocabulary.csv',
+    './extended_vocab.csv',
+    './domain_vocab.csv'
+];
+const SHARD_SIZE = 100 * 1024; // 100KB
+const OUTPUT_DIR = path.join(__dirname, 'vocabShards');
+const EMBEDDINGS_API = process.env.EMBEDDINGS_API || 'http://localhost:5000/api/embed';
+const DOMAINS = [
+    'general', 'science', 'technology', 'medicine', 'engineering',
+    'art', 'history', 'geography', 'music', 'literature',
+    'business', 'finance', 'sports', 'politics', 'education',
+    'psychology', 'philosophy', 'religion', 'food', 'travel'
+];
+
+// 确保输出目录存在
+if (!fs.existsSync(OUTPUT_DIR)) {
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+}
+
+async function getWordEmbedding(word) {
+    try {
+        const response = await axios.post(EMBEDDINGS_API, { text: word });
+        return response.data.embedding;
+    } catch (error) {
+        console.error(`获取词嵌入失败: ${word}`, error.message);
+        // 回退策略: 生成基于词的哈希的伪嵌入
+        const hash = createHash('sha256').update(word).digest('hex');
+        const pseudoEmbedding = Array.from({ length: 32 }, (_, i) => 
+            parseInt(hash.substring(i*2, i*2+2), 16) / 255 - 0.5
+        );
+        return pseudoEmbedding;
+    }
+}
+
+async function loadAllVocabulary() {
+    console.log('加载所有词表...');
+    const allWords = new Set();
+    
+    for (const file of VOCAB_FILES) {
+        const filePath = path.join(__dirname, file);
+        if (!fs.existsSync(filePath)) {
+            console.warn(`警告: 词表文件不存在 ${filePath}`);
+            continue;
+        }
+        
+        const fileStream = fs.createReadStream(filePath);
+        const rl = readline.createInterface({
+            input: fileStream,
+            crlfDelay: Infinity
+        });
+        
+        for await (const line of rl) {
+            const word = line.trim().split(',')[0]; // 假设第一列是词
+            if (word && word.length > 1) {
+                allWords.add(word);
+            }
+        }
+    }
+    
+    console.log(`总共加载了 ${allWords.size} 个唯一词条`);
+    return Array.from(allWords);
+}
+
+async function assignWordsToDomains(words) {
+    console.log('将词语分配到领域...');
+    const domainWords = {};
+    DOMAINS.forEach(domain => { domainWords[domain] = []; });
+    
+    // 如果词太多，简单起见，我们可以按字母哈希分配
+    if (words.length > 10000) {
+        console.log('词条太多，使用简单哈希进行分配...');
+        words.forEach(word => {
+            const hash = word.charCodeAt(0) % DOMAINS.length;
+            domainWords[DOMAINS[hash]].push(word);
+        });
+    } else {
+        // 对于较小的词表，我们可以尝试使用词嵌入聚类
+        console.log('使用词嵌入进行语义聚类...');
+        
+        const wordsWithEmbeddings = [];
+        // 每1000个词显示进度
+        const progressInterval = Math.max(1, Math.floor(words.length / 10));
+        
+        for (let i = 0; i < words.length; i++) {
+            const word = words[i];
+            const embedding = await getWordEmbedding(word);
+            wordsWithEmbeddings.push({ word, embedding });
+            
+            if ((i + 1) % progressInterval === 0 || i === words.length - 1) {
+                console.log(`进度: ${i+1}/${words.length} (${((i+1)/words.length*100).toFixed(1)}%)`);
+            }
+        }
+        
+        // 简单K-means聚类
+        const domainCentroids = await initializeCentroids(wordsWithEmbeddings, DOMAINS.length);
+        
+        // 分配词到最近的质心
+        wordsWithEmbeddings.forEach(({ word, embedding }) => {
+            let minDist = Infinity;
+            let bestDomain = DOMAINS[0];
+            
+            for (let i = 0; i < domainCentroids.length; i++) {
+                const dist = euclideanDistance(embedding, domainCentroids[i]);
+                if (dist < minDist) {
+                    minDist = dist;
+                    bestDomain = DOMAINS[i];
+                }
+            }
+            
+            domainWords[bestDomain].push(word);
+        });
+    }
+    
+    // 报告每个领域的词数
+    for (const domain of DOMAINS) {
+        console.log(`${domain}: ${domainWords[domain].length} 词`);
+    }
+    
+    return domainWords;
+}
+
+function euclideanDistance(vec1, vec2) {
+    let sum = 0;
+    for (let i = 0; i < vec1.length; i++) {
+        sum += Math.pow(vec1[i] - vec2[i], 2);
+    }
+    return Math.sqrt(sum);
+}
+
+async function initializeCentroids(wordsWithEmbeddings, k) {
+    // 简单的质心初始化: 随机选择k个点
+    const shuffled = [...wordsWithEmbeddings].sort(() => 0.5 - Math.random());
+    return shuffled.slice(0, k).map(w => w.embedding);
+}
+
+function splitIntoShards(domainWords) {
+    console.log('将词分成固定大小的分片...');
+    const shards = [];
+    
+    for (const domain of DOMAINS) {
+        const words = domainWords[domain];
+        let currentShard = [];
+        let currentSize = 0;
+        let shardCount = 0;
+        
+        words.forEach(word => {
+            // 估计这个词占用的字节数 (简单估计，utf8编码下每个字符1-4字节)
+            const wordSize = Buffer.from(word).length + 1; // +1 是换行符
+            
+            if (currentSize + wordSize > SHARD_SIZE && currentShard.length > 0) {
+                // 保存当前分片
+                shards.push({
+                    domain,
+                    index: shardCount++,
+                    words: currentShard,
+                    size: currentSize
+                });
+                
+                // 开始新分片
+                currentShard = [word];
+                currentSize = wordSize;
+            } else {
+                currentShard.push(word);
+                currentSize += wordSize;
+            }
+        });
+        
+        // 不要忘记最后一个分片
+        if (currentShard.length > 0) {
+            shards.push({
+                domain,
+                index: shardCount++,
+                words: currentShard,
+                size: currentSize
+            });
+        }
+    }
+    
+    console.log(`总共创建了 ${shards.length} 个分片`);
+    return shards;
+}
+
+function saveShards(shards) {
+    console.log('保存分片到文件...');
+    
+    shards.forEach((shard, idx) => {
+        const filename = `${shard.domain}_${shard.index}.csv`;
+        const filePath = path.join(OUTPUT_DIR, filename);
+        
+        const content = shard.words.join('\n');
+        fs.writeFileSync(filePath, content);
+        
+        console.log(`已保存: ${filename} (${shard.words.length} 词, ${Math.round(shard.size/1024)} KB)`);
+    });
+    
+    // 创建分片索引文件
+    const shardIndex = shards.map((shard, idx) => ({
+        id: idx,
+        filename: `${shard.domain}_${shard.index}.csv`,
+        domain: shard.domain,
+        wordCount: shard.words.length,
+        sizeKB: Math.round(shard.size/1024)
+    }));
+    
+    fs.writeFileSync(
+        path.join(OUTPUT_DIR, 'shard_index.json'), 
+        JSON.stringify(shardIndex, null, 2)
+    );
+}
+
+async function main() {
+    try {
+        console.log('开始词表分片...');
+        // 1. 加载所有词表
+        const allWords = await loadAllVocabulary();
+        
+        // 2. 将词分配到不同领域
+        const domainWords = await assignWordsToDomains(allWords);
+        
+        // 3. 将每个领域的词分成固定大小的分片
+        const shards = splitIntoShards(domainWords);
+        
+        // 4. 保存分片
+        saveShards(shards);
+        
+        console.log('词表分片完成！');
+    } catch (error) {
+        console.error('词表分片失败:', error);
+        process.exit(1);
+    }
+}
+
+// 启动主流程
+if (require.main === module) {
+    main();
+}
+
+module.exports = {
+    loadAllVocabulary,
+    assignWordsToDomains,
+    splitIntoShards,
+    saveShards
+};

package/wikitext/.gitattributes

@@ -0,0 +1,27 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

package/wikitext/README.md

@@ -0,0 +1,344 @@
+---
+annotations_creators:
+- no-annotation
+language_creators:
+- crowdsourced
+language:
+- en
+license:
+- cc-by-sa-3.0
+- gfdl
+multilinguality:
+- monolingual
+size_categories:
+- 1M<n<10M
+source_datasets:
+- original
+task_categories:
+- text-generation
+- fill-mask
+task_ids:
+- language-modeling
+- masked-language-modeling
+paperswithcode_id: wikitext-2
+pretty_name: WikiText
+dataset_info:
+- config_name: wikitext-103-raw-v1
+  features:
+  - name: text
+    dtype: string
+  splits:
+  - name: test
+    num_bytes: 1305088
+    num_examples: 4358
+  - name: train
+    num_bytes: 546500949
+    num_examples: 1801350
+  - name: validation
+    num_bytes: 1159288
+    num_examples: 3760
+  download_size: 315466397
+  dataset_size: 548965325
+- config_name: wikitext-103-v1
+  features:
+  - name: text
+    dtype: string
+  splits:
+  - name: test
+    num_bytes: 1295575
+    num_examples: 4358
+  - name: train
+    num_bytes: 545141915
+    num_examples: 1801350
+  - name: validation
+    num_bytes: 1154751
+    num_examples: 3760
+  download_size: 313093838
+  dataset_size: 547592241
+- config_name: wikitext-2-raw-v1
+  features:
+  - name: text
+    dtype: string
+  splits:
+  - name: test
+    num_bytes: 1305088
+    num_examples: 4358
+  - name: train
+    num_bytes: 11061717
+    num_examples: 36718
+  - name: validation
+    num_bytes: 1159288
+    num_examples: 3760
+  download_size: 7747362
+  dataset_size: 13526093
+- config_name: wikitext-2-v1
+  features:
+  - name: text
+    dtype: string
+  splits:
+  - name: test
+    num_bytes: 1270947
+    num_examples: 4358
+  - name: train
+    num_bytes: 10918118
+    num_examples: 36718
+  - name: validation
+    num_bytes: 1134123
+    num_examples: 3760
+  download_size: 7371282
+  dataset_size: 13323188
+configs:
+- config_name: wikitext-103-raw-v1
+  data_files:
+  - split: test
+    path: wikitext-103-raw-v1/test-*
+  - split: train
+    path: wikitext-103-raw-v1/train-*
+  - split: validation
+    path: wikitext-103-raw-v1/validation-*
+- config_name: wikitext-103-v1
+  data_files:
+  - split: test
+    path: wikitext-103-v1/test-*
+  - split: train
+    path: wikitext-103-v1/train-*
+  - split: validation
+    path: wikitext-103-v1/validation-*
+- config_name: wikitext-2-raw-v1
+  data_files:
+  - split: test
+    path: wikitext-2-raw-v1/test-*
+  - split: train
+    path: wikitext-2-raw-v1/train-*
+  - split: validation
+    path: wikitext-2-raw-v1/validation-*
+- config_name: wikitext-2-v1
+  data_files:
+  - split: test
+    path: wikitext-2-v1/test-*
+  - split: train
+    path: wikitext-2-v1/train-*
+  - split: validation
+    path: wikitext-2-v1/validation-*
+---
+
+# Dataset Card for "wikitext"
+
+## Table of Contents
+- [Dataset Description](#dataset-description)
+  - [Dataset Summary](#dataset-summary)
+  - [Supported Tasks and Leaderboards](#supported-tasks-and-leaderboards)
+  - [Languages](#languages)
+- [Dataset Structure](#dataset-structure)
+  - [Data Instances](#data-instances)
+  - [Data Fields](#data-fields)
+  - [Data Splits](#data-splits)
+- [Dataset Creation](#dataset-creation)
+  - [Curation Rationale](#curation-rationale)
+  - [Source Data](#source-data)
+  - [Annotations](#annotations)
+  - [Personal and Sensitive Information](#personal-and-sensitive-information)
+- [Considerations for Using the Data](#considerations-for-using-the-data)
+  - [Social Impact of Dataset](#social-impact-of-dataset)
+  - [Discussion of Biases](#discussion-of-biases)
+  - [Other Known Limitations](#other-known-limitations)
+- [Additional Information](#additional-information)
+  - [Dataset Curators](#dataset-curators)
+  - [Licensing Information](#licensing-information)
+  - [Citation Information](#citation-information)
+  - [Contributions](#contributions)
+
+## Dataset Description
+
+- **Homepage:** [https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/)
+- **Repository:** [More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)
+- **Paper:** [Pointer Sentinel Mixture Models](https://arxiv.org/abs/1609.07843)
+- **Point of Contact:** [Stephen Merity](mailto:smerity@salesforce.com)
+- **Size of downloaded dataset files:** 391.41 MB
+- **Size of the generated dataset:** 1.12 GB
+- **Total amount of disk used:** 1.52 GB
+
+### Dataset Summary
+
+ The WikiText language modeling dataset is a collection of over 100 million tokens extracted from the set of verified
+ Good and Featured articles on Wikipedia. The dataset is available under the Creative Commons Attribution-ShareAlike License.
+
+Compared to the preprocessed version of Penn Treebank (PTB), WikiText-2 is over 2 times larger and WikiText-103 is over
+110 times larger. The WikiText dataset also features a far larger vocabulary and retains the original case, punctuation
+and numbers - all of which are removed in PTB. As it is composed of full articles, the dataset is well suited for models
+that can take advantage of long term dependencies.
+
+Each subset comes in two different variants:
+- Raw (for character level work) contain the raw tokens, before the addition of the <unk> (unknown) tokens.
+- Non-raw (for word level work) contain only the tokens in their vocabulary (wiki.train.tokens, wiki.valid.tokens, and wiki.test.tokens).
+  The out-of-vocabulary tokens have been replaced with the the <unk> token.
+
+
+### Supported Tasks and Leaderboards
+
+[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)
+
+### Languages
+
+[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)
+
+## Dataset Structure
+
+### Data Instances
+
+#### wikitext-103-raw-v1
+
+- **Size of downloaded dataset files:** 191.98 MB
+- **Size of the generated dataset:** 549.42 MB
+- **Total amount of disk used:** 741.41 MB
+
+An example of 'validation' looks as follows.
+```
+This example was too long and was cropped:
+
+{
+    "text": "\" The gold dollar or gold one @-@ dollar piece was a coin struck as a regular issue by the United States Bureau of the Mint from..."
+}
+```
+
+#### wikitext-103-v1
+
+- **Size of downloaded dataset files:** 190.23 MB
+- **Size of the generated dataset:** 548.05 MB
+- **Total amount of disk used:** 738.27 MB
+
+An example of 'train' looks as follows.
+```
+This example was too long and was cropped:
+
+{
+    "text": "\" Senjō no Valkyria 3 : <unk> Chronicles ( Japanese : 戦場のヴァルキュリア3 , lit . Valkyria of the Battlefield 3 ) , commonly referred to..."
+}
+```
+
+#### wikitext-2-raw-v1
+
+- **Size of downloaded dataset files:** 4.72 MB
+- **Size of the generated dataset:** 13.54 MB
+- **Total amount of disk used:** 18.26 MB
+
+An example of 'train' looks as follows.
+```
+This example was too long and was cropped:
+
+{
+    "text": "\" The Sinclair Scientific Programmable was introduced in 1975 , with the same case as the Sinclair Oxford . It was larger than t..."
+}
+```
+
+#### wikitext-2-v1
+
+- **Size of downloaded dataset files:** 4.48 MB
+- **Size of the generated dataset:** 13.34 MB
+- **Total amount of disk used:** 17.82 MB
+
+An example of 'train' looks as follows.
+```
+This example was too long and was cropped:
+
+{
+    "text": "\" Senjō no Valkyria 3 : <unk> Chronicles ( Japanese : 戦場のヴァルキュリア3 , lit . Valkyria of the Battlefield 3 ) , commonly referred to..."
+}
+```
+
+### Data Fields
+
+The data fields are the same among all splits.
+
+#### wikitext-103-raw-v1
+- `text`: a `string` feature.
+
+#### wikitext-103-v1
+- `text`: a `string` feature.
+
+#### wikitext-2-raw-v1
+- `text`: a `string` feature.
+
+#### wikitext-2-v1
+- `text`: a `string` feature.
+
+### Data Splits
+
+|       name        | train |validation|test|
+|-------------------|------:|---------:|---:|
+|wikitext-103-raw-v1|1801350|      3760|4358|
+|wikitext-103-v1    |1801350|      3760|4358|
+|wikitext-2-raw-v1  |  36718|      3760|4358|
+|wikitext-2-v1      |  36718|      3760|4358|
+
+## Dataset Creation
+
+### Curation Rationale
+
+[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)
+
+### Source Data
+
+#### Initial Data Collection and Normalization
+
+[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)
+
+#### Who are the source language producers?
+
+[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)
+
+### Annotations
+
+#### Annotation process
+
+[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)
+
+#### Who are the annotators?
+
+[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)
+
+### Personal and Sensitive Information
+
+[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)
+
+## Considerations for Using the Data
+
+### Social Impact of Dataset
+
+[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)
+
+### Discussion of Biases
+
+[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)
+
+### Other Known Limitations
+
+[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)
+
+## Additional Information
+
+### Dataset Curators
+
+[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)
+
+### Licensing Information
+
+The dataset is available under the [Creative Commons Attribution-ShareAlike License (CC BY-SA 4.0)](https://creativecommons.org/licenses/by-sa/4.0/).
+
+### Citation Information
+
+```
+@misc{merity2016pointer,
+      title={Pointer Sentinel Mixture Models},
+      author={Stephen Merity and Caiming Xiong and James Bradbury and Richard Socher},
+      year={2016},
+      eprint={1609.07843},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```
+
+
+### Contributions
+
+Thanks to [@thomwolf](https://github.com/thomwolf), [@lewtun](https://github.com/lewtun), [@patrickvonplaten](https://github.com/patrickvonplaten), [@mariamabarham](https://github.com/mariamabarham) for adding this dataset.

package/wikitext/describtion.txt

@@ -0,0 +1 @@
+说明：这个仓库来自https://hf-mirror.com/datasets/Salesforce/wikitext