@melaya/runner 1.0.43 → 1.0.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/dist/connection.js +14 -0
  2. package/nltk_data/tokenizers/punkt/PY3/README +98 -0
  3. package/nltk_data/tokenizers/punkt/PY3/english.pickle +0 -0
  4. package/nltk_data/tokenizers/punkt_tab/english/abbrev_types.txt +156 -0
  5. package/nltk_data/tokenizers/punkt_tab/english/collocations.tab +37 -0
  6. package/nltk_data/tokenizers/punkt_tab/english/ortho_context.tab +20366 -0
  7. package/nltk_data/tokenizers/punkt_tab/english/sent_starters.txt +39 -0
  8. package/package.json +2 -1
package/dist/connection.js CHANGED
@@ -529,6 +529,19 @@ export async function connect(opts) {
529
529
  // lives.
530
530
  const storePath = join(homedir(), ".melaya-runner", "rag", payload.pipelineName, "store");
531
531
  console.log(chalk.gray(` [rag] store → ${storePath}`));
532
+ // NLTK_DATA points at the English-only tokenizer bundle shipped
533
+ // inside this npm package (`<runner>/nltk_data/`). Hermetic — no
534
+ // network, no SSL/proxy quirks, no `Resource 'punkt_tab' not found`.
535
+ // The runner published 1.0.43 and earlier relied on `nltk.download`
536
+ // at runtime which silently no-op'd on some macOS Pythons (cert
537
+ // verification path issue). Bundling the ~657KB English bundle
538
+ // eliminates the failure mode entirely.
539
+ const candidateNltkPaths = [
540
+ join(__dirname, "..", "nltk_data"),
541
+ join(__dirname, "nltk_data"),
542
+ ];
543
+ const { existsSync: _exists2 } = await import("fs");
544
+ const bundledNltkData = candidateNltkPaths.find((p) => _exists2(p)) || "";
532
545
  const proc = spawn(env.pythonPath, [
533
546
  "-u", stagedScript,
534
547
  "--pipeline-name", payload.pipelineName,
@@ -539,6 +552,7 @@ export async function connect(opts) {
539
552
  env: {
540
553
  ...process.env,
541
554
  PYTHONPATH: sharedDir, // so `from agentscope.rag import …` resolves
555
+ ...(bundledNltkData ? { NLTK_DATA: bundledNltkData } : {}),
542
556
  },
543
557
  cwd: workDir,
544
558
  });
package/nltk_data/tokenizers/punkt/PY3/README ADDED
@@ -0,0 +1,98 @@
1
+ Pretrained Punkt Models -- Jan Strunk (New version trained after issues 313 and 514 had been corrected)
2
+
3
+ Most models were prepared using the test corpora from Kiss and Strunk (2006). Additional models have
4
+ been contributed by various people using NLTK for sentence boundary detection.
5
+
6
+ For information about how to use these models, please confer the tokenization HOWTO:
7
+ http://nltk.googlecode.com/svn/trunk/doc/howto/tokenize.html
8
+ and chapter 3.8 of the NLTK book:
9
+ http://nltk.googlecode.com/svn/trunk/doc/book/ch03.html#sec-segmentation
10
+
11
+ There are pretrained tokenizers for the following languages:
12
+
13
+ File Language Source Contents Size of training corpus(in tokens) Model contributed by
14
+ =======================================================================================================================================================================
15
+ czech.pickle Czech Multilingual Corpus 1 (ECI) Lidove Noviny ~345,000 Jan Strunk / Tibor Kiss
16
+ Literarni Noviny
17
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
18
+ danish.pickle Danish Avisdata CD-Rom Ver. 1.1. 1995 Berlingske Tidende ~550,000 Jan Strunk / Tibor Kiss
19
+ (Berlingske Avisdata, Copenhagen) Weekend Avisen
20
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
21
+ dutch.pickle Dutch Multilingual Corpus 1 (ECI) De Limburger ~340,000 Jan Strunk / Tibor Kiss
22
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
23
+ english.pickle English Penn Treebank (LDC) Wall Street Journal ~469,000 Jan Strunk / Tibor Kiss
24
+ (American)
25
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
26
+ estonian.pickle Estonian University of Tartu, Estonia Eesti Ekspress ~359,000 Jan Strunk / Tibor Kiss
27
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
28
+ finnish.pickle Finnish Finnish Parole Corpus, Finnish Books and major national ~364,000 Jan Strunk / Tibor Kiss
29
+ Text Bank (Suomen Kielen newspapers
30
+ Tekstipankki)
31
+ Finnish Center for IT Science
32
+ (CSC)
33
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
34
+ french.pickle French Multilingual Corpus 1 (ECI) Le Monde ~370,000 Jan Strunk / Tibor Kiss
35
+ (European)
36
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
37
+ german.pickle German Neue Zürcher Zeitung AG Neue Zürcher Zeitung ~847,000 Jan Strunk / Tibor Kiss
38
+ (Switzerland) CD-ROM
39
+ (Uses "ss"
40
+ instead of "ß")
41
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
42
+ greek.pickle Greek Efstathios Stamatatos To Vima (TO BHMA) ~227,000 Jan Strunk / Tibor Kiss
43
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
44
+ italian.pickle Italian Multilingual Corpus 1 (ECI) La Stampa, Il Mattino ~312,000 Jan Strunk / Tibor Kiss
45
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
46
+ norwegian.pickle Norwegian Centre for Humanities Bergens Tidende ~479,000 Jan Strunk / Tibor Kiss
47
+ (Bokmål and Information Technologies,
48
+ Nynorsk) Bergen
49
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
50
+ polish.pickle Polish Polish National Corpus Literature, newspapers, etc. ~1,000,000 Krzysztof Langner
51
+ (http://www.nkjp.pl/)
52
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
53
+ portuguese.pickle Portuguese CETENFolha Corpus Folha de São Paulo ~321,000 Jan Strunk / Tibor Kiss
54
+ (Brazilian) (Linguateca)
55
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
56
+ slovene.pickle Slovene TRACTOR Delo ~354,000 Jan Strunk / Tibor Kiss
57
+ Slovene Academy for Arts
58
+ and Sciences
59
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
60
+ spanish.pickle Spanish Multilingual Corpus 1 (ECI) Sur ~353,000 Jan Strunk / Tibor Kiss
61
+ (European)
62
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
63
+ swedish.pickle Swedish Multilingual Corpus 1 (ECI) Dagens Nyheter ~339,000 Jan Strunk / Tibor Kiss
64
+ (and some other texts)
65
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
66
+ turkish.pickle Turkish METU Turkish Corpus Milliyet ~333,000 Jan Strunk / Tibor Kiss
67
+ (Türkçe Derlem Projesi)
68
+ University of Ankara
69
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
70
+
71
+ The corpora contained about 400,000 tokens on average and mostly consisted of newspaper text converted to
72
+ Unicode using the codecs module.
73
+
74
+ Kiss, Tibor and Strunk, Jan (2006): Unsupervised Multilingual Sentence Boundary Detection.
75
+ Computational Linguistics 32: 485-525.
76
+
77
+ ---- Training Code ----
78
+
79
+ # import punkt
80
+ import nltk.tokenize.punkt
81
+
82
+ # Make a new Tokenizer
83
+ tokenizer = nltk.tokenize.punkt.PunktSentenceTokenizer()
84
+
85
+ # Read in training corpus (one example: Slovene)
86
+ import codecs
87
+ text = codecs.open("slovene.plain","Ur","iso-8859-2").read()
88
+
89
+ # Train tokenizer
90
+ tokenizer.train(text)
91
+
92
+ # Dump pickled tokenizer
93
+ import pickle
94
+ out = open("slovene.pickle","wb")
95
+ pickle.dump(tokenizer, out)
96
+ out.close()
97
+
98
+ ---------
package/nltk_data/tokenizers/punkt_tab/english/abbrev_types.txt ADDED
@@ -0,0 +1,156 @@
1
+ ct
2
+ m.j
3
+ t
4
+ a.c
5
+ n.h
6
+ ms
7
+ p.a.m
8
+ dr
9
+ pa
10
+ p.m
11
+ u.k
12
+ st
13
+ dec
14
+ u.s.a
15
+ lt
16
+ g.k
17
+ adm
18
+ p
19
+ h.m
20
+ ga
21
+ tenn
22
+ yr
23
+ sen
24
+ n.c
25
+ j.j
26
+ d.h
27
+ s.g
28
+ inc
29
+ vs
30
+ s.p.a
31
+ a.t
32
+ n
33
+ feb
34
+ sr
35
+ jan
36
+ s.a.y
37
+ n.y
38
+ col
39
+ g.f
40
+ c.o.m.b
41
+ d
42
+ ft
43
+ va
44
+ r.k
45
+ e.f
46
+ chg
47
+ r.i
48
+ a.g
49
+ minn
50
+ a.h
51
+ k
52
+ n.j
53
+ m
54
+ l.f
55
+ f.j
56
+ gen
57
+ i.m.s
58
+ s.a
59
+ aug
60
+ j.p
61
+ okla
62
+ m.d.c
63
+ ltd
64
+ oct
65
+ s
66
+ vt
67
+ r.a
68
+ j.c
69
+ ariz
70
+ w.w
71
+ b.v
72
+ ore
73
+ h
74
+ w.r
75
+ e.h
76
+ mrs
77
+ cie
78
+ corp
79
+ w
80
+ n.v
81
+ a.d
82
+ r.j
83
+ ok
84
+ . .
85
+ e.m
86
+ w.c
87
+ ill
88
+ nov
89
+ u.s
90
+ prof
91
+ conn
92
+ u.s.s.r
93
+ mg
94
+ f.g
95
+ ph.d
96
+ g
97
+ calif
98
+ messrs
99
+ h.f
100
+ wash
101
+ tues
102
+ sw
103
+ bros
104
+ u.n
105
+ l
106
+ wis
107
+ mr
108
+ sep
109
+ d.c
110
+ ave
111
+ e.l
112
+ co
113
+ s.s
114
+ reps
115
+ c
116
+ r.t
117
+ h.c
118
+ r
119
+ wed
120
+ a.s
121
+ v
122
+ fla
123
+ jr
124
+ r.h
125
+ c.v
126
+ m.b.a
127
+ rep
128
+ a.a
129
+ e
130
+ c.i.t
131
+ l.a
132
+ b.f
133
+ j.b
134
+ d.w
135
+ j.k
136
+ ala
137
+ f
138
+ w.va
139
+ sept
140
+ mich
141
+ n.m
142
+ j.r
143
+ l.p
144
+ s.c
145
+ colo
146
+ fri
147
+ a.m
148
+ g.d
149
+ kan
150
+ maj
151
+ ky
152
+ a.m.e
153
+ n.d
154
+ t.j
155
+ cos
156
+ nev
package/nltk_data/tokenizers/punkt_tab/english/collocations.tab ADDED
@@ -0,0 +1,37 @@
1
+ ##number## international
2
+ ##number## rj
3
+ ##number## commodities
4
+ ##number## cooper
5
+ b stewart
6
+ ##number## genentech
7
+ ##number## wedgestone
8
+ i toussie
9
+ ##number## pepper
10
+ j fialka
11
+ o ludcke
12
+ ##number## insider
13
+ ##number## aes
14
+ i magnin
15
+ ##number## credit
16
+ ##number## corrections
17
+ ##number## financing
18
+ ##number## henley
19
+ ##number## business
20
+ ##number## pay-fone
21
+ b wigton
22
+ b edelman
23
+ b levine
24
+ ##number## leisure
25
+ b smith
26
+ j walter
27
+ ##number## pegasus
28
+ ##number## dividend
29
+ j aron
30
+ ##number## review
31
+ ##number## abreast
32
+ ##number## who
33
+ ##number## letters
34
+ ##number## colgate
35
+ ##number## cbot
36
+ ##number## notable
37
+ ##number## zimmer