ruby-stemmer-dimelo 0.9.3.dimelo1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. checksums.yaml +7 -0
  2. data/MIT-LICENSE +21 -0
  3. data/README.rdoc +117 -0
  4. data/Rakefile +70 -0
  5. data/VERSION +1 -0
  6. data/ext/lingua/extconf.rb +40 -0
  7. data/ext/lingua/stemmer.c +115 -0
  8. data/lib/lingua/stemmer.rb +60 -0
  9. data/libstemmer_c/MANIFEST +72 -0
  10. data/libstemmer_c/Makefile +9 -0
  11. data/libstemmer_c/Makefile.windows +15 -0
  12. data/libstemmer_c/README +125 -0
  13. data/libstemmer_c/examples/stemwords.c +209 -0
  14. data/libstemmer_c/include/libstemmer.h +79 -0
  15. data/libstemmer_c/libstemmer/libstemmer.c +93 -0
  16. data/libstemmer_c/libstemmer/libstemmer_utf8.c +93 -0
  17. data/libstemmer_c/libstemmer/modules.h +195 -0
  18. data/libstemmer_c/libstemmer/modules.txt +51 -0
  19. data/libstemmer_c/libstemmer/modules_utf8.h +123 -0
  20. data/libstemmer_c/libstemmer/modules_utf8.txt +50 -0
  21. data/libstemmer_c/mkinc.mak +86 -0
  22. data/libstemmer_c/mkinc_utf8.mak +54 -0
  23. data/libstemmer_c/runtime/api.c +66 -0
  24. data/libstemmer_c/runtime/api.h +26 -0
  25. data/libstemmer_c/runtime/header.h +58 -0
  26. data/libstemmer_c/runtime/utilities.c +478 -0
  27. data/libstemmer_c/src_c/stem_ISO_8859_1_danish.c +337 -0
  28. data/libstemmer_c/src_c/stem_ISO_8859_1_danish.h +16 -0
  29. data/libstemmer_c/src_c/stem_ISO_8859_1_dutch.c +624 -0
  30. data/libstemmer_c/src_c/stem_ISO_8859_1_dutch.h +16 -0
  31. data/libstemmer_c/src_c/stem_ISO_8859_1_english.c +1117 -0
  32. data/libstemmer_c/src_c/stem_ISO_8859_1_english.h +16 -0
  33. data/libstemmer_c/src_c/stem_ISO_8859_1_finnish.c +762 -0
  34. data/libstemmer_c/src_c/stem_ISO_8859_1_finnish.h +16 -0
  35. data/libstemmer_c/src_c/stem_ISO_8859_1_french.c +1230 -0
  36. data/libstemmer_c/src_c/stem_ISO_8859_1_french.h +16 -0
  37. data/libstemmer_c/src_c/stem_ISO_8859_1_german.c +503 -0
  38. data/libstemmer_c/src_c/stem_ISO_8859_1_german.h +16 -0
  39. data/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.c +1230 -0
  40. data/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.h +16 -0
  41. data/libstemmer_c/src_c/stem_ISO_8859_1_italian.c +1065 -0
  42. data/libstemmer_c/src_c/stem_ISO_8859_1_italian.h +16 -0
  43. data/libstemmer_c/src_c/stem_ISO_8859_1_latin.c +443 -0
  44. data/libstemmer_c/src_c/stem_ISO_8859_1_latin.h +16 -0
  45. data/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.c +297 -0
  46. data/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.h +16 -0
  47. data/libstemmer_c/src_c/stem_ISO_8859_1_porter.c +749 -0
  48. data/libstemmer_c/src_c/stem_ISO_8859_1_porter.h +16 -0
  49. data/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.c +1017 -0
  50. data/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.h +16 -0
  51. data/libstemmer_c/src_c/stem_ISO_8859_1_spanish.c +1093 -0
  52. data/libstemmer_c/src_c/stem_ISO_8859_1_spanish.h +16 -0
  53. data/libstemmer_c/src_c/stem_ISO_8859_1_swedish.c +307 -0
  54. data/libstemmer_c/src_c/stem_ISO_8859_1_swedish.h +16 -0
  55. data/libstemmer_c/src_c/stem_ISO_8859_2_romanian.c +998 -0
  56. data/libstemmer_c/src_c/stem_ISO_8859_2_romanian.h +16 -0
  57. data/libstemmer_c/src_c/stem_KOI8_R_russian.c +700 -0
  58. data/libstemmer_c/src_c/stem_KOI8_R_russian.h +16 -0
  59. data/libstemmer_c/src_c/stem_UTF_8_danish.c +339 -0
  60. data/libstemmer_c/src_c/stem_UTF_8_danish.h +16 -0
  61. data/libstemmer_c/src_c/stem_UTF_8_dutch.c +634 -0
  62. data/libstemmer_c/src_c/stem_UTF_8_dutch.h +16 -0
  63. data/libstemmer_c/src_c/stem_UTF_8_english.c +1125 -0
  64. data/libstemmer_c/src_c/stem_UTF_8_english.h +16 -0
  65. data/libstemmer_c/src_c/stem_UTF_8_finnish.c +768 -0
  66. data/libstemmer_c/src_c/stem_UTF_8_finnish.h +16 -0
  67. data/libstemmer_c/src_c/stem_UTF_8_french.c +1230 -0
  68. data/libstemmer_c/src_c/stem_UTF_8_french.h +16 -0
  69. data/libstemmer_c/src_c/stem_UTF_8_german.c +509 -0
  70. data/libstemmer_c/src_c/stem_UTF_8_german.h +16 -0
  71. data/libstemmer_c/src_c/stem_UTF_8_hungarian.c +1234 -0
  72. data/libstemmer_c/src_c/stem_UTF_8_hungarian.h +16 -0
  73. data/libstemmer_c/src_c/stem_UTF_8_italian.c +1073 -0
  74. data/libstemmer_c/src_c/stem_UTF_8_italian.h +16 -0
  75. data/libstemmer_c/src_c/stem_UTF_8_latin.c +443 -0
  76. data/libstemmer_c/src_c/stem_UTF_8_latin.h +16 -0
  77. data/libstemmer_c/src_c/stem_UTF_8_norwegian.c +299 -0
  78. data/libstemmer_c/src_c/stem_UTF_8_norwegian.h +16 -0
  79. data/libstemmer_c/src_c/stem_UTF_8_porter.c +755 -0
  80. data/libstemmer_c/src_c/stem_UTF_8_porter.h +16 -0
  81. data/libstemmer_c/src_c/stem_UTF_8_portuguese.c +1023 -0
  82. data/libstemmer_c/src_c/stem_UTF_8_portuguese.h +16 -0
  83. data/libstemmer_c/src_c/stem_UTF_8_romanian.c +1004 -0
  84. data/libstemmer_c/src_c/stem_UTF_8_romanian.h +16 -0
  85. data/libstemmer_c/src_c/stem_UTF_8_russian.c +694 -0
  86. data/libstemmer_c/src_c/stem_UTF_8_russian.h +16 -0
  87. data/libstemmer_c/src_c/stem_UTF_8_spanish.c +1097 -0
  88. data/libstemmer_c/src_c/stem_UTF_8_spanish.h +16 -0
  89. data/libstemmer_c/src_c/stem_UTF_8_swedish.c +309 -0
  90. data/libstemmer_c/src_c/stem_UTF_8_swedish.h +16 -0
  91. data/libstemmer_c/src_c/stem_UTF_8_turkish.c +2205 -0
  92. data/libstemmer_c/src_c/stem_UTF_8_turkish.h +16 -0
  93. data/test/helper.rb +3 -0
  94. data/test/lingua/test_stemmer.rb +99 -0
  95. metadata +141 -0
@@ -0,0 +1,50 @@
1
+ # This file contains a list of stemmers to include in the distribution.
2
+ # The format is a set of space separated lines - on each line:
3
+ # First item is name of stemmer.
4
+ # Second item is comma separated list of character sets.
5
+ # Third item is comma separated list of names to refer to the stemmer by.
6
+ #
7
+ # Lines starting with a #, or blank lines, are ignored.
8
+
9
+ # List all the main algorithms for each language, in UTF-8.
10
+
11
+ latin UTF_8 latin
12
+ danish UTF_8 danish,da,dan
13
+ dutch UTF_8 dutch,nl,dut,nld
14
+ english UTF_8 english,en,eng
15
+ finnish UTF_8 finnish,fi,fin
16
+ french UTF_8 french,fr,fre,fra
17
+ german UTF_8 german,de,ger,deu
18
+ hungarian UTF_8 hungarian,hu,hun
19
+ italian UTF_8 italian,it,ita
20
+ norwegian UTF_8 norwegian,no,nor
21
+ portuguese UTF_8 portuguese,pt,por
22
+ romanian UTF_8 romanian,ro,rum,ron
23
+ russian UTF_8 russian,ru,rus
24
+ spanish UTF_8 spanish,es,esl,spa
25
+ swedish UTF_8 swedish,sv,swe
26
+ turkish UTF_8 turkish,tr,tur
27
+
28
+ # Also include the traditional porter algorithm for english.
29
+ # The porter algorithm is included in the libstemmer distribution to assist
30
+ # with backwards compatibility, but for new systems the english algorithm
31
+ # should be used in preference.
32
+ porter UTF_8 porter
33
+
34
+ # Some other stemmers in the snowball project are not included in the standard
35
+ # distribution. To compile a libstemmer with them in, add them to this list,
36
+ # and regenerate the distribution. (You will need a full source checkout for
37
+ # this.) They are included in the snowball website as curiosities, but are not
38
+ # intended for general use, and use of them is is not fully supported. These
39
+ # algorithms are:
40
+ #
41
+ # german2 - This is a slight modification of the german stemmer.
42
+ #german2 UTF_8 german2
43
+ #
44
+ # kraaij_pohlmann - This is a different dutch stemmer.
45
+ #kraaij_pohlmann UTF_8 kraaij_pohlmann
46
+ #
47
+ # lovins - This is an english stemmer, but fairly outdated, and
48
+ # only really applicable to a restricted type of input text
49
+ # (keywords in academic publications).
50
+ #lovins UTF_8 lovins
@@ -0,0 +1,86 @@
1
+ # libstemmer/mkinc.mak: List of stemming module source files
2
+ #
3
+ # This file is generated by mkmodules.pl from a list of module names.
4
+ # Do not edit manually.
5
+ #
6
+ # Modules included by this file are: latin, danish, dutch, english, finnish, french,
7
+ # german, hungarian, italian, norwegian, porter, portuguese, romanian,
8
+ # russian, spanish, swedish, turkish
9
+
10
+ snowball_sources= \
11
+ src_c/stem_ISO_8859_1_latin.c \
12
+ src_c/stem_UTF_8_latin.c \
13
+ src_c/stem_ISO_8859_1_danish.c \
14
+ src_c/stem_UTF_8_danish.c \
15
+ src_c/stem_ISO_8859_1_dutch.c \
16
+ src_c/stem_UTF_8_dutch.c \
17
+ src_c/stem_ISO_8859_1_english.c \
18
+ src_c/stem_UTF_8_english.c \
19
+ src_c/stem_ISO_8859_1_finnish.c \
20
+ src_c/stem_UTF_8_finnish.c \
21
+ src_c/stem_ISO_8859_1_french.c \
22
+ src_c/stem_UTF_8_french.c \
23
+ src_c/stem_ISO_8859_1_german.c \
24
+ src_c/stem_UTF_8_german.c \
25
+ src_c/stem_ISO_8859_1_hungarian.c \
26
+ src_c/stem_UTF_8_hungarian.c \
27
+ src_c/stem_ISO_8859_1_italian.c \
28
+ src_c/stem_UTF_8_italian.c \
29
+ src_c/stem_ISO_8859_1_norwegian.c \
30
+ src_c/stem_UTF_8_norwegian.c \
31
+ src_c/stem_ISO_8859_1_porter.c \
32
+ src_c/stem_UTF_8_porter.c \
33
+ src_c/stem_ISO_8859_1_portuguese.c \
34
+ src_c/stem_UTF_8_portuguese.c \
35
+ src_c/stem_ISO_8859_2_romanian.c \
36
+ src_c/stem_UTF_8_romanian.c \
37
+ src_c/stem_KOI8_R_russian.c \
38
+ src_c/stem_UTF_8_russian.c \
39
+ src_c/stem_ISO_8859_1_spanish.c \
40
+ src_c/stem_UTF_8_spanish.c \
41
+ src_c/stem_ISO_8859_1_swedish.c \
42
+ src_c/stem_UTF_8_swedish.c \
43
+ src_c/stem_UTF_8_turkish.c \
44
+ runtime/api.c \
45
+ runtime/utilities.c \
46
+ libstemmer/libstemmer.c
47
+
48
+ snowball_headers= \
49
+ src_c/stem_ISO_8859_1_latin.h \
50
+ src_c/stem_UTF_8_latin.h \
51
+ src_c/stem_ISO_8859_1_danish.h \
52
+ src_c/stem_UTF_8_danish.h \
53
+ src_c/stem_ISO_8859_1_dutch.h \
54
+ src_c/stem_UTF_8_dutch.h \
55
+ src_c/stem_ISO_8859_1_english.h \
56
+ src_c/stem_UTF_8_english.h \
57
+ src_c/stem_ISO_8859_1_finnish.h \
58
+ src_c/stem_UTF_8_finnish.h \
59
+ src_c/stem_ISO_8859_1_french.h \
60
+ src_c/stem_UTF_8_french.h \
61
+ src_c/stem_ISO_8859_1_german.h \
62
+ src_c/stem_UTF_8_german.h \
63
+ src_c/stem_ISO_8859_1_hungarian.h \
64
+ src_c/stem_UTF_8_hungarian.h \
65
+ src_c/stem_ISO_8859_1_italian.h \
66
+ src_c/stem_UTF_8_italian.h \
67
+ src_c/stem_ISO_8859_1_norwegian.h \
68
+ src_c/stem_UTF_8_norwegian.h \
69
+ src_c/stem_ISO_8859_1_porter.h \
70
+ src_c/stem_UTF_8_porter.h \
71
+ src_c/stem_ISO_8859_1_portuguese.h \
72
+ src_c/stem_UTF_8_portuguese.h \
73
+ src_c/stem_ISO_8859_2_romanian.h \
74
+ src_c/stem_UTF_8_romanian.h \
75
+ src_c/stem_KOI8_R_russian.h \
76
+ src_c/stem_UTF_8_russian.h \
77
+ src_c/stem_ISO_8859_1_spanish.h \
78
+ src_c/stem_UTF_8_spanish.h \
79
+ src_c/stem_ISO_8859_1_swedish.h \
80
+ src_c/stem_UTF_8_swedish.h \
81
+ src_c/stem_UTF_8_turkish.h \
82
+ include/libstemmer.h \
83
+ libstemmer/modules.h \
84
+ runtime/api.h \
85
+ runtime/header.h
86
+
@@ -0,0 +1,54 @@
1
+ # libstemmer/mkinc_utf8.mak: List of stemming module source files
2
+ #
3
+ # This file is generated by mkmodules.pl from a list of module names.
4
+ # Do not edit manually.
5
+ #
6
+ # Modules included by this file are: danish, dutch, english, finnish, french,
7
+ # german, hungarian, italian, norwegian, porter, portuguese, romanian,
8
+ # russian, spanish, swedish, turkish
9
+
10
+ snowball_sources= \
11
+ src_c/stem_UTF_8_latin.c \
12
+ src_c/stem_UTF_8_danish.c \
13
+ src_c/stem_UTF_8_dutch.c \
14
+ src_c/stem_UTF_8_english.c \
15
+ src_c/stem_UTF_8_finnish.c \
16
+ src_c/stem_UTF_8_french.c \
17
+ src_c/stem_UTF_8_german.c \
18
+ src_c/stem_UTF_8_hungarian.c \
19
+ src_c/stem_UTF_8_italian.c \
20
+ src_c/stem_UTF_8_norwegian.c \
21
+ src_c/stem_UTF_8_porter.c \
22
+ src_c/stem_UTF_8_portuguese.c \
23
+ src_c/stem_UTF_8_romanian.c \
24
+ src_c/stem_UTF_8_russian.c \
25
+ src_c/stem_UTF_8_spanish.c \
26
+ src_c/stem_UTF_8_swedish.c \
27
+ src_c/stem_UTF_8_turkish.c \
28
+ runtime/api.c \
29
+ runtime/utilities.c \
30
+ libstemmer/libstemmer_utf8.c
31
+
32
+ snowball_headers= \
33
+ src_c/stem_UTF_8_latin.h \
34
+ src_c/stem_UTF_8_danish.h \
35
+ src_c/stem_UTF_8_dutch.h \
36
+ src_c/stem_UTF_8_english.h \
37
+ src_c/stem_UTF_8_finnish.h \
38
+ src_c/stem_UTF_8_french.h \
39
+ src_c/stem_UTF_8_german.h \
40
+ src_c/stem_UTF_8_hungarian.h \
41
+ src_c/stem_UTF_8_italian.h \
42
+ src_c/stem_UTF_8_norwegian.h \
43
+ src_c/stem_UTF_8_porter.h \
44
+ src_c/stem_UTF_8_portuguese.h \
45
+ src_c/stem_UTF_8_romanian.h \
46
+ src_c/stem_UTF_8_russian.h \
47
+ src_c/stem_UTF_8_spanish.h \
48
+ src_c/stem_UTF_8_swedish.h \
49
+ src_c/stem_UTF_8_turkish.h \
50
+ include/libstemmer.h \
51
+ libstemmer/modules_utf8.h \
52
+ runtime/api.h \
53
+ runtime/header.h
54
+
@@ -0,0 +1,66 @@
1
+
2
+ #include <stdlib.h> /* for calloc, free */
3
+ #include "header.h"
4
+
5
+ extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size)
6
+ {
7
+ struct SN_env * z = (struct SN_env *) calloc(1, sizeof(struct SN_env));
8
+ if (z == NULL) return NULL;
9
+ z->p = create_s();
10
+ if (z->p == NULL) goto error;
11
+ if (S_size)
12
+ {
13
+ int i;
14
+ z->S = (symbol * *) calloc(S_size, sizeof(symbol *));
15
+ if (z->S == NULL) goto error;
16
+
17
+ for (i = 0; i < S_size; i++)
18
+ {
19
+ z->S[i] = create_s();
20
+ if (z->S[i] == NULL) goto error;
21
+ }
22
+ }
23
+
24
+ if (I_size)
25
+ {
26
+ z->I = (int *) calloc(I_size, sizeof(int));
27
+ if (z->I == NULL) goto error;
28
+ }
29
+
30
+ if (B_size)
31
+ {
32
+ z->B = (unsigned char *) calloc(B_size, sizeof(unsigned char));
33
+ if (z->B == NULL) goto error;
34
+ }
35
+
36
+ return z;
37
+ error:
38
+ SN_close_env(z, S_size);
39
+ return NULL;
40
+ }
41
+
42
+ extern void SN_close_env(struct SN_env * z, int S_size)
43
+ {
44
+ if (z == NULL) return;
45
+ if (S_size)
46
+ {
47
+ int i;
48
+ for (i = 0; i < S_size; i++)
49
+ {
50
+ lose_s(z->S[i]);
51
+ }
52
+ free(z->S);
53
+ }
54
+ free(z->I);
55
+ free(z->B);
56
+ if (z->p) lose_s(z->p);
57
+ free(z);
58
+ }
59
+
60
+ extern int SN_set_current(struct SN_env * z, int size, const symbol * s)
61
+ {
62
+ int err = replace_s(z, 0, z->l, size, s, NULL);
63
+ z->c = 0;
64
+ return err;
65
+ }
66
+
@@ -0,0 +1,26 @@
1
+
2
+ typedef unsigned char symbol;
3
+
4
+ /* Or replace 'char' above with 'short' for 16 bit characters.
5
+
6
+ More precisely, replace 'char' with whatever type guarantees the
7
+ character width you need. Note however that sizeof(symbol) should divide
8
+ HEAD, defined in header.h as 2*sizeof(int), without remainder, otherwise
9
+ there is an alignment problem. In the unlikely event of a problem here,
10
+ consult Martin Porter.
11
+
12
+ */
13
+
14
+ struct SN_env {
15
+ symbol * p;
16
+ int c; int l; int lb; int bra; int ket;
17
+ symbol * * S;
18
+ int * I;
19
+ unsigned char * B;
20
+ };
21
+
22
+ extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size);
23
+ extern void SN_close_env(struct SN_env * z, int S_size);
24
+
25
+ extern int SN_set_current(struct SN_env * z, int size, const symbol * s);
26
+
@@ -0,0 +1,58 @@
1
+
2
+ #include <limits.h>
3
+
4
+ #include "api.h"
5
+
6
+ #define MAXINT INT_MAX
7
+ #define MININT INT_MIN
8
+
9
+ #define HEAD 2*sizeof(int)
10
+
11
+ #define SIZE(p) ((int *)(p))[-1]
12
+ #define SET_SIZE(p, n) ((int *)(p))[-1] = n
13
+ #define CAPACITY(p) ((int *)(p))[-2]
14
+
15
+ struct among
16
+ { int s_size; /* number of chars in string */
17
+ const symbol * s; /* search string */
18
+ int substring_i;/* index to longest matching substring */
19
+ int result; /* result of the lookup */
20
+ int (* function)(struct SN_env *);
21
+ };
22
+
23
+ extern symbol * create_s(void);
24
+ extern void lose_s(symbol * p);
25
+
26
+ extern int skip_utf8(const symbol * p, int c, int lb, int l, int n);
27
+
28
+ extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
29
+ extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
30
+ extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
31
+ extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
32
+
33
+ extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
34
+ extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
35
+ extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
36
+ extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
37
+
38
+ extern int eq_s(struct SN_env * z, int s_size, const symbol * s);
39
+ extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s);
40
+ extern int eq_v(struct SN_env * z, const symbol * p);
41
+ extern int eq_v_b(struct SN_env * z, const symbol * p);
42
+
43
+ extern int find_among(struct SN_env * z, const struct among * v, int v_size);
44
+ extern int find_among_b(struct SN_env * z, const struct among * v, int v_size);
45
+
46
+ extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjustment);
47
+ extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s);
48
+ extern int slice_from_v(struct SN_env * z, const symbol * p);
49
+ extern int slice_del(struct SN_env * z);
50
+
51
+ extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s);
52
+ extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p);
53
+
54
+ extern symbol * slice_to(struct SN_env * z, symbol * p);
55
+ extern symbol * assign_to(struct SN_env * z, symbol * p);
56
+
57
+ extern void debug(struct SN_env * z, int number, int line_count);
58
+
@@ -0,0 +1,478 @@
1
+
2
+ #include <stdio.h>
3
+ #include <stdlib.h>
4
+ #include <string.h>
5
+
6
+ #include "header.h"
7
+
8
+ #define unless(C) if(!(C))
9
+
10
+ #define CREATE_SIZE 1
11
+
12
+ extern symbol * create_s(void) {
13
+ symbol * p;
14
+ void * mem = malloc(HEAD + (CREATE_SIZE + 1) * sizeof(symbol));
15
+ if (mem == NULL) return NULL;
16
+ p = (symbol *) (HEAD + (char *) mem);
17
+ CAPACITY(p) = CREATE_SIZE;
18
+ SET_SIZE(p, CREATE_SIZE);
19
+ return p;
20
+ }
21
+
22
+ extern void lose_s(symbol * p) {
23
+ if (p == NULL) return;
24
+ free((char *) p - HEAD);
25
+ }
26
+
27
+ /*
28
+ new_p = skip_utf8(p, c, lb, l, n); skips n characters forwards from p + c
29
+ if n +ve, or n characters backwards from p + c - 1 if n -ve. new_p is the new
30
+ position, or 0 on failure.
31
+
32
+ -- used to implement hop and next in the utf8 case.
33
+ */
34
+
35
+ extern int skip_utf8(const symbol * p, int c, int lb, int l, int n) {
36
+ int b;
37
+ if (n >= 0) {
38
+ for (; n > 0; n--) {
39
+ if (c >= l) return -1;
40
+ b = p[c++];
41
+ if (b >= 0xC0) { /* 1100 0000 */
42
+ while (c < l) {
43
+ b = p[c];
44
+ if (b >= 0xC0 || b < 0x80) break;
45
+ /* break unless b is 10------ */
46
+ c++;
47
+ }
48
+ }
49
+ }
50
+ } else {
51
+ for (; n < 0; n++) {
52
+ if (c <= lb) return -1;
53
+ b = p[--c];
54
+ if (b >= 0x80) { /* 1000 0000 */
55
+ while (c > lb) {
56
+ b = p[c];
57
+ if (b >= 0xC0) break; /* 1100 0000 */
58
+ c--;
59
+ }
60
+ }
61
+ }
62
+ }
63
+ return c;
64
+ }
65
+
66
+ /* Code for character groupings: utf8 cases */
67
+
68
+ static int get_utf8(const symbol * p, int c, int l, int * slot) {
69
+ int b0, b1;
70
+ if (c >= l) return 0;
71
+ b0 = p[c++];
72
+ if (b0 < 0xC0 || c == l) { /* 1100 0000 */
73
+ * slot = b0; return 1;
74
+ }
75
+ b1 = p[c++];
76
+ if (b0 < 0xE0 || c == l) { /* 1110 0000 */
77
+ * slot = (b0 & 0x1F) << 6 | (b1 & 0x3F); return 2;
78
+ }
79
+ * slot = (b0 & 0xF) << 12 | (b1 & 0x3F) << 6 | (p[c] & 0x3F); return 3;
80
+ }
81
+
82
+ static int get_b_utf8(const symbol * p, int c, int lb, int * slot) {
83
+ int b0, b1;
84
+ if (c <= lb) return 0;
85
+ b0 = p[--c];
86
+ if (b0 < 0x80 || c == lb) { /* 1000 0000 */
87
+ * slot = b0; return 1;
88
+ }
89
+ b1 = p[--c];
90
+ if (b1 >= 0xC0 || c == lb) { /* 1100 0000 */
91
+ * slot = (b1 & 0x1F) << 6 | (b0 & 0x3F); return 2;
92
+ }
93
+ * slot = (p[c] & 0xF) << 12 | (b1 & 0x3F) << 6 | (b0 & 0x3F); return 3;
94
+ }
95
+
96
+ extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
97
+ do {
98
+ int ch;
99
+ int w = get_utf8(z->p, z->c, z->l, & ch);
100
+ unless (w) return -1;
101
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
102
+ return w;
103
+ z->c += w;
104
+ } while (repeat);
105
+ return 0;
106
+ }
107
+
108
+ extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
109
+ do {
110
+ int ch;
111
+ int w = get_b_utf8(z->p, z->c, z->lb, & ch);
112
+ unless (w) return -1;
113
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
114
+ return w;
115
+ z->c -= w;
116
+ } while (repeat);
117
+ return 0;
118
+ }
119
+
120
+ extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
121
+ do {
122
+ int ch;
123
+ int w = get_utf8(z->p, z->c, z->l, & ch);
124
+ unless (w) return -1;
125
+ unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
126
+ return w;
127
+ z->c += w;
128
+ } while (repeat);
129
+ return 0;
130
+ }
131
+
132
+ extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
133
+ do {
134
+ int ch;
135
+ int w = get_b_utf8(z->p, z->c, z->lb, & ch);
136
+ unless (w) return -1;
137
+ unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
138
+ return w;
139
+ z->c -= w;
140
+ } while (repeat);
141
+ return 0;
142
+ }
143
+
144
+ /* Code for character groupings: non-utf8 cases */
145
+
146
+ extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
147
+ do {
148
+ int ch;
149
+ if (z->c >= z->l) return -1;
150
+ ch = z->p[z->c];
151
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
152
+ return 1;
153
+ z->c++;
154
+ } while (repeat);
155
+ return 0;
156
+ }
157
+
158
+ extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
159
+ do {
160
+ int ch;
161
+ if (z->c <= z->lb) return -1;
162
+ ch = z->p[z->c - 1];
163
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
164
+ return 1;
165
+ z->c--;
166
+ } while (repeat);
167
+ return 0;
168
+ }
169
+
170
+ extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
171
+ do {
172
+ int ch;
173
+ if (z->c >= z->l) return -1;
174
+ ch = z->p[z->c];
175
+ unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
176
+ return 1;
177
+ z->c++;
178
+ } while (repeat);
179
+ return 0;
180
+ }
181
+
182
+ extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
183
+ do {
184
+ int ch;
185
+ if (z->c <= z->lb) return -1;
186
+ ch = z->p[z->c - 1];
187
+ unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
188
+ return 1;
189
+ z->c--;
190
+ } while (repeat);
191
+ return 0;
192
+ }
193
+
194
+ extern int eq_s(struct SN_env * z, int s_size, const symbol * s) {
195
+ if (z->l - z->c < s_size || memcmp(z->p + z->c, s, s_size * sizeof(symbol)) != 0) return 0;
196
+ z->c += s_size; return 1;
197
+ }
198
+
199
+ extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s) {
200
+ if (z->c - z->lb < s_size || memcmp(z->p + z->c - s_size, s, s_size * sizeof(symbol)) != 0) return 0;
201
+ z->c -= s_size; return 1;
202
+ }
203
+
204
+ extern int eq_v(struct SN_env * z, const symbol * p) {
205
+ return eq_s(z, SIZE(p), p);
206
+ }
207
+
208
+ extern int eq_v_b(struct SN_env * z, const symbol * p) {
209
+ return eq_s_b(z, SIZE(p), p);
210
+ }
211
+
212
+ extern int find_among(struct SN_env * z, const struct among * v, int v_size) {
213
+
214
+ int i = 0;
215
+ int j = v_size;
216
+
217
+ int c = z->c; int l = z->l;
218
+ symbol * q = z->p + c;
219
+
220
+ const struct among * w;
221
+
222
+ int common_i = 0;
223
+ int common_j = 0;
224
+
225
+ int first_key_inspected = 0;
226
+
227
+ while(1) {
228
+ int k = i + ((j - i) >> 1);
229
+ int diff = 0;
230
+ int common = common_i < common_j ? common_i : common_j; /* smaller */
231
+ w = v + k;
232
+ {
233
+ int i2; for (i2 = common; i2 < w->s_size; i2++) {
234
+ if (c + common == l) { diff = -1; break; }
235
+ diff = q[common] - w->s[i2];
236
+ if (diff != 0) break;
237
+ common++;
238
+ }
239
+ }
240
+ if (diff < 0) { j = k; common_j = common; }
241
+ else { i = k; common_i = common; }
242
+ if (j - i <= 1) {
243
+ if (i > 0) break; /* v->s has been inspected */
244
+ if (j == i) break; /* only one item in v */
245
+
246
+ /* - but now we need to go round once more to get
247
+ v->s inspected. This looks messy, but is actually
248
+ the optimal approach. */
249
+
250
+ if (first_key_inspected) break;
251
+ first_key_inspected = 1;
252
+ }
253
+ }
254
+ while(1) {
255
+ w = v + i;
256
+ if (common_i >= w->s_size) {
257
+ z->c = c + w->s_size;
258
+ if (w->function == 0) return w->result;
259
+ {
260
+ int res = w->function(z);
261
+ z->c = c + w->s_size;
262
+ if (res) return w->result;
263
+ }
264
+ }
265
+ i = w->substring_i;
266
+ if (i < 0) return 0;
267
+ }
268
+ }
269
+
270
+ /* find_among_b is for backwards processing. Same comments apply */
271
+
272
+ extern int find_among_b(struct SN_env * z, const struct among * v, int v_size) {
273
+
274
+ int i = 0;
275
+ int j = v_size;
276
+
277
+ int c = z->c; int lb = z->lb;
278
+ symbol * q = z->p + c - 1;
279
+
280
+ const struct among * w;
281
+
282
+ int common_i = 0;
283
+ int common_j = 0;
284
+
285
+ int first_key_inspected = 0;
286
+
287
+ while(1) {
288
+ int k = i + ((j - i) >> 1);
289
+ int diff = 0;
290
+ int common = common_i < common_j ? common_i : common_j;
291
+ w = v + k;
292
+ {
293
+ int i2; for (i2 = w->s_size - 1 - common; i2 >= 0; i2--) {
294
+ if (c - common == lb) { diff = -1; break; }
295
+ diff = q[- common] - w->s[i2];
296
+ if (diff != 0) break;
297
+ common++;
298
+ }
299
+ }
300
+ if (diff < 0) { j = k; common_j = common; }
301
+ else { i = k; common_i = common; }
302
+ if (j - i <= 1) {
303
+ if (i > 0) break;
304
+ if (j == i) break;
305
+ if (first_key_inspected) break;
306
+ first_key_inspected = 1;
307
+ }
308
+ }
309
+ while(1) {
310
+ w = v + i;
311
+ if (common_i >= w->s_size) {
312
+ z->c = c - w->s_size;
313
+ if (w->function == 0) return w->result;
314
+ {
315
+ int res = w->function(z);
316
+ z->c = c - w->s_size;
317
+ if (res) return w->result;
318
+ }
319
+ }
320
+ i = w->substring_i;
321
+ if (i < 0) return 0;
322
+ }
323
+ }
324
+
325
+
326
+ /* Increase the size of the buffer pointed to by p to at least n symbols.
327
+ * If insufficient memory, returns NULL and frees the old buffer.
328
+ */
329
+ static symbol * increase_size(symbol * p, int n) {
330
+ symbol * q;
331
+ int new_size = n + 20;
332
+ void * mem = realloc((char *) p - HEAD,
333
+ HEAD + (new_size + 1) * sizeof(symbol));
334
+ if (mem == NULL) {
335
+ lose_s(p);
336
+ return NULL;
337
+ }
338
+ q = (symbol *) (HEAD + (char *)mem);
339
+ CAPACITY(q) = new_size;
340
+ return q;
341
+ }
342
+
343
+ /* to replace symbols between c_bra and c_ket in z->p by the
344
+ s_size symbols at s.
345
+ Returns 0 on success, -1 on error.
346
+ Also, frees z->p (and sets it to NULL) on error.
347
+ */
348
+ extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjptr)
349
+ {
350
+ int adjustment;
351
+ int len;
352
+ if (z->p == NULL) {
353
+ z->p = create_s();
354
+ if (z->p == NULL) return -1;
355
+ }
356
+ adjustment = s_size - (c_ket - c_bra);
357
+ len = SIZE(z->p);
358
+ if (adjustment != 0) {
359
+ if (adjustment + len > CAPACITY(z->p)) {
360
+ z->p = increase_size(z->p, adjustment + len);
361
+ if (z->p == NULL) return -1;
362
+ }
363
+ memmove(z->p + c_ket + adjustment,
364
+ z->p + c_ket,
365
+ (len - c_ket) * sizeof(symbol));
366
+ SET_SIZE(z->p, adjustment + len);
367
+ z->l += adjustment;
368
+ if (z->c >= c_ket)
369
+ z->c += adjustment;
370
+ else
371
+ if (z->c > c_bra)
372
+ z->c = c_bra;
373
+ }
374
+ unless (s_size == 0) memmove(z->p + c_bra, s, s_size * sizeof(symbol));
375
+ if (adjptr != NULL)
376
+ *adjptr = adjustment;
377
+ return 0;
378
+ }
379
+
380
+ static int slice_check(struct SN_env * z) {
381
+
382
+ if (z->bra < 0 ||
383
+ z->bra > z->ket ||
384
+ z->ket > z->l ||
385
+ z->p == NULL ||
386
+ z->l > SIZE(z->p)) /* this line could be removed */
387
+ {
388
+ #if 0
389
+ fprintf(stderr, "faulty slice operation:\n");
390
+ debug(z, -1, 0);
391
+ #endif
392
+ return -1;
393
+ }
394
+ return 0;
395
+ }
396
+
397
+ extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s) {
398
+ if (slice_check(z)) return -1;
399
+ return replace_s(z, z->bra, z->ket, s_size, s, NULL);
400
+ }
401
+
402
+ extern int slice_from_v(struct SN_env * z, const symbol * p) {
403
+ return slice_from_s(z, SIZE(p), p);
404
+ }
405
+
406
+ extern int slice_del(struct SN_env * z) {
407
+ return slice_from_s(z, 0, 0);
408
+ }
409
+
410
+ extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s) {
411
+ int adjustment;
412
+ if (replace_s(z, bra, ket, s_size, s, &adjustment))
413
+ return -1;
414
+ if (bra <= z->bra) z->bra += adjustment;
415
+ if (bra <= z->ket) z->ket += adjustment;
416
+ return 0;
417
+ }
418
+
419
+ extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p) {
420
+ int adjustment;
421
+ if (replace_s(z, bra, ket, SIZE(p), p, &adjustment))
422
+ return -1;
423
+ if (bra <= z->bra) z->bra += adjustment;
424
+ if (bra <= z->ket) z->ket += adjustment;
425
+ return 0;
426
+ }
427
+
428
+ extern symbol * slice_to(struct SN_env * z, symbol * p) {
429
+ if (slice_check(z)) {
430
+ lose_s(p);
431
+ return NULL;
432
+ }
433
+ {
434
+ int len = z->ket - z->bra;
435
+ if (CAPACITY(p) < len) {
436
+ p = increase_size(p, len);
437
+ if (p == NULL)
438
+ return NULL;
439
+ }
440
+ memmove(p, z->p + z->bra, len * sizeof(symbol));
441
+ SET_SIZE(p, len);
442
+ }
443
+ return p;
444
+ }
445
+
446
+ extern symbol * assign_to(struct SN_env * z, symbol * p) {
447
+ int len = z->l;
448
+ if (CAPACITY(p) < len) {
449
+ p = increase_size(p, len);
450
+ if (p == NULL)
451
+ return NULL;
452
+ }
453
+ memmove(p, z->p, len * sizeof(symbol));
454
+ SET_SIZE(p, len);
455
+ return p;
456
+ }
457
+
458
+ #if 0
459
+ extern void debug(struct SN_env * z, int number, int line_count) {
460
+ int i;
461
+ int limit = SIZE(z->p);
462
+ /*if (number >= 0) printf("%3d (line %4d): '", number, line_count);*/
463
+ if (number >= 0) printf("%3d (line %4d): [%d]'", number, line_count,limit);
464
+ for (i = 0; i <= limit; i++) {
465
+ if (z->lb == i) printf("{");
466
+ if (z->bra == i) printf("[");
467
+ if (z->c == i) printf("|");
468
+ if (z->ket == i) printf("]");
469
+ if (z->l == i) printf("}");
470
+ if (i < limit)
471
+ { int ch = z->p[i];
472
+ if (ch == 0) ch = '#';
473
+ printf("%c", ch);
474
+ }
475
+ }
476
+ printf("'\n");
477
+ }
478
+ #endif