ruby-stemmer-dimelo 0.9.3.dimelo1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (95) hide show
  1. checksums.yaml +7 -0
  2. data/MIT-LICENSE +21 -0
  3. data/README.rdoc +117 -0
  4. data/Rakefile +70 -0
  5. data/VERSION +1 -0
  6. data/ext/lingua/extconf.rb +40 -0
  7. data/ext/lingua/stemmer.c +115 -0
  8. data/lib/lingua/stemmer.rb +60 -0
  9. data/libstemmer_c/MANIFEST +72 -0
  10. data/libstemmer_c/Makefile +9 -0
  11. data/libstemmer_c/Makefile.windows +15 -0
  12. data/libstemmer_c/README +125 -0
  13. data/libstemmer_c/examples/stemwords.c +209 -0
  14. data/libstemmer_c/include/libstemmer.h +79 -0
  15. data/libstemmer_c/libstemmer/libstemmer.c +93 -0
  16. data/libstemmer_c/libstemmer/libstemmer_utf8.c +93 -0
  17. data/libstemmer_c/libstemmer/modules.h +195 -0
  18. data/libstemmer_c/libstemmer/modules.txt +51 -0
  19. data/libstemmer_c/libstemmer/modules_utf8.h +123 -0
  20. data/libstemmer_c/libstemmer/modules_utf8.txt +50 -0
  21. data/libstemmer_c/mkinc.mak +86 -0
  22. data/libstemmer_c/mkinc_utf8.mak +54 -0
  23. data/libstemmer_c/runtime/api.c +66 -0
  24. data/libstemmer_c/runtime/api.h +26 -0
  25. data/libstemmer_c/runtime/header.h +58 -0
  26. data/libstemmer_c/runtime/utilities.c +478 -0
  27. data/libstemmer_c/src_c/stem_ISO_8859_1_danish.c +337 -0
  28. data/libstemmer_c/src_c/stem_ISO_8859_1_danish.h +16 -0
  29. data/libstemmer_c/src_c/stem_ISO_8859_1_dutch.c +624 -0
  30. data/libstemmer_c/src_c/stem_ISO_8859_1_dutch.h +16 -0
  31. data/libstemmer_c/src_c/stem_ISO_8859_1_english.c +1117 -0
  32. data/libstemmer_c/src_c/stem_ISO_8859_1_english.h +16 -0
  33. data/libstemmer_c/src_c/stem_ISO_8859_1_finnish.c +762 -0
  34. data/libstemmer_c/src_c/stem_ISO_8859_1_finnish.h +16 -0
  35. data/libstemmer_c/src_c/stem_ISO_8859_1_french.c +1230 -0
  36. data/libstemmer_c/src_c/stem_ISO_8859_1_french.h +16 -0
  37. data/libstemmer_c/src_c/stem_ISO_8859_1_german.c +503 -0
  38. data/libstemmer_c/src_c/stem_ISO_8859_1_german.h +16 -0
  39. data/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.c +1230 -0
  40. data/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.h +16 -0
  41. data/libstemmer_c/src_c/stem_ISO_8859_1_italian.c +1065 -0
  42. data/libstemmer_c/src_c/stem_ISO_8859_1_italian.h +16 -0
  43. data/libstemmer_c/src_c/stem_ISO_8859_1_latin.c +443 -0
  44. data/libstemmer_c/src_c/stem_ISO_8859_1_latin.h +16 -0
  45. data/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.c +297 -0
  46. data/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.h +16 -0
  47. data/libstemmer_c/src_c/stem_ISO_8859_1_porter.c +749 -0
  48. data/libstemmer_c/src_c/stem_ISO_8859_1_porter.h +16 -0
  49. data/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.c +1017 -0
  50. data/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.h +16 -0
  51. data/libstemmer_c/src_c/stem_ISO_8859_1_spanish.c +1093 -0
  52. data/libstemmer_c/src_c/stem_ISO_8859_1_spanish.h +16 -0
  53. data/libstemmer_c/src_c/stem_ISO_8859_1_swedish.c +307 -0
  54. data/libstemmer_c/src_c/stem_ISO_8859_1_swedish.h +16 -0
  55. data/libstemmer_c/src_c/stem_ISO_8859_2_romanian.c +998 -0
  56. data/libstemmer_c/src_c/stem_ISO_8859_2_romanian.h +16 -0
  57. data/libstemmer_c/src_c/stem_KOI8_R_russian.c +700 -0
  58. data/libstemmer_c/src_c/stem_KOI8_R_russian.h +16 -0
  59. data/libstemmer_c/src_c/stem_UTF_8_danish.c +339 -0
  60. data/libstemmer_c/src_c/stem_UTF_8_danish.h +16 -0
  61. data/libstemmer_c/src_c/stem_UTF_8_dutch.c +634 -0
  62. data/libstemmer_c/src_c/stem_UTF_8_dutch.h +16 -0
  63. data/libstemmer_c/src_c/stem_UTF_8_english.c +1125 -0
  64. data/libstemmer_c/src_c/stem_UTF_8_english.h +16 -0
  65. data/libstemmer_c/src_c/stem_UTF_8_finnish.c +768 -0
  66. data/libstemmer_c/src_c/stem_UTF_8_finnish.h +16 -0
  67. data/libstemmer_c/src_c/stem_UTF_8_french.c +1230 -0
  68. data/libstemmer_c/src_c/stem_UTF_8_french.h +16 -0
  69. data/libstemmer_c/src_c/stem_UTF_8_german.c +509 -0
  70. data/libstemmer_c/src_c/stem_UTF_8_german.h +16 -0
  71. data/libstemmer_c/src_c/stem_UTF_8_hungarian.c +1234 -0
  72. data/libstemmer_c/src_c/stem_UTF_8_hungarian.h +16 -0
  73. data/libstemmer_c/src_c/stem_UTF_8_italian.c +1073 -0
  74. data/libstemmer_c/src_c/stem_UTF_8_italian.h +16 -0
  75. data/libstemmer_c/src_c/stem_UTF_8_latin.c +443 -0
  76. data/libstemmer_c/src_c/stem_UTF_8_latin.h +16 -0
  77. data/libstemmer_c/src_c/stem_UTF_8_norwegian.c +299 -0
  78. data/libstemmer_c/src_c/stem_UTF_8_norwegian.h +16 -0
  79. data/libstemmer_c/src_c/stem_UTF_8_porter.c +755 -0
  80. data/libstemmer_c/src_c/stem_UTF_8_porter.h +16 -0
  81. data/libstemmer_c/src_c/stem_UTF_8_portuguese.c +1023 -0
  82. data/libstemmer_c/src_c/stem_UTF_8_portuguese.h +16 -0
  83. data/libstemmer_c/src_c/stem_UTF_8_romanian.c +1004 -0
  84. data/libstemmer_c/src_c/stem_UTF_8_romanian.h +16 -0
  85. data/libstemmer_c/src_c/stem_UTF_8_russian.c +694 -0
  86. data/libstemmer_c/src_c/stem_UTF_8_russian.h +16 -0
  87. data/libstemmer_c/src_c/stem_UTF_8_spanish.c +1097 -0
  88. data/libstemmer_c/src_c/stem_UTF_8_spanish.h +16 -0
  89. data/libstemmer_c/src_c/stem_UTF_8_swedish.c +309 -0
  90. data/libstemmer_c/src_c/stem_UTF_8_swedish.h +16 -0
  91. data/libstemmer_c/src_c/stem_UTF_8_turkish.c +2205 -0
  92. data/libstemmer_c/src_c/stem_UTF_8_turkish.h +16 -0
  93. data/test/helper.rb +3 -0
  94. data/test/lingua/test_stemmer.rb +99 -0
  95. metadata +141 -0
@@ -0,0 +1,50 @@
1
+ # This file contains a list of stemmers to include in the distribution.
2
+ # The format is a set of space separated lines - on each line:
3
+ # First item is name of stemmer.
4
+ # Second item is comma separated list of character sets.
5
+ # Third item is comma separated list of names to refer to the stemmer by.
6
+ #
7
+ # Lines starting with a #, or blank lines, are ignored.
8
+
9
+ # List all the main algorithms for each language, in UTF-8.
10
+
11
+ latin UTF_8 latin
12
+ danish UTF_8 danish,da,dan
13
+ dutch UTF_8 dutch,nl,dut,nld
14
+ english UTF_8 english,en,eng
15
+ finnish UTF_8 finnish,fi,fin
16
+ french UTF_8 french,fr,fre,fra
17
+ german UTF_8 german,de,ger,deu
18
+ hungarian UTF_8 hungarian,hu,hun
19
+ italian UTF_8 italian,it,ita
20
+ norwegian UTF_8 norwegian,no,nor
21
+ portuguese UTF_8 portuguese,pt,por
22
+ romanian UTF_8 romanian,ro,rum,ron
23
+ russian UTF_8 russian,ru,rus
24
+ spanish UTF_8 spanish,es,esl,spa
25
+ swedish UTF_8 swedish,sv,swe
26
+ turkish UTF_8 turkish,tr,tur
27
+
28
+ # Also include the traditional porter algorithm for english.
29
+ # The porter algorithm is included in the libstemmer distribution to assist
30
+ # with backwards compatibility, but for new systems the english algorithm
31
+ # should be used in preference.
32
+ porter UTF_8 porter
33
+
34
+ # Some other stemmers in the snowball project are not included in the standard
35
+ # distribution. To compile a libstemmer with them in, add them to this list,
36
+ # and regenerate the distribution. (You will need a full source checkout for
37
+ # this.) They are included in the snowball website as curiosities, but are not
38
+ # intended for general use, and use of them is is not fully supported. These
39
+ # algorithms are:
40
+ #
41
+ # german2 - This is a slight modification of the german stemmer.
42
+ #german2 UTF_8 german2
43
+ #
44
+ # kraaij_pohlmann - This is a different dutch stemmer.
45
+ #kraaij_pohlmann UTF_8 kraaij_pohlmann
46
+ #
47
+ # lovins - This is an english stemmer, but fairly outdated, and
48
+ # only really applicable to a restricted type of input text
49
+ # (keywords in academic publications).
50
+ #lovins UTF_8 lovins
@@ -0,0 +1,86 @@
1
+ # libstemmer/mkinc.mak: List of stemming module source files
2
+ #
3
+ # This file is generated by mkmodules.pl from a list of module names.
4
+ # Do not edit manually.
5
+ #
6
+ # Modules included by this file are: latin, danish, dutch, english, finnish, french,
7
+ # german, hungarian, italian, norwegian, porter, portuguese, romanian,
8
+ # russian, spanish, swedish, turkish
9
+
10
+ snowball_sources= \
11
+ src_c/stem_ISO_8859_1_latin.c \
12
+ src_c/stem_UTF_8_latin.c \
13
+ src_c/stem_ISO_8859_1_danish.c \
14
+ src_c/stem_UTF_8_danish.c \
15
+ src_c/stem_ISO_8859_1_dutch.c \
16
+ src_c/stem_UTF_8_dutch.c \
17
+ src_c/stem_ISO_8859_1_english.c \
18
+ src_c/stem_UTF_8_english.c \
19
+ src_c/stem_ISO_8859_1_finnish.c \
20
+ src_c/stem_UTF_8_finnish.c \
21
+ src_c/stem_ISO_8859_1_french.c \
22
+ src_c/stem_UTF_8_french.c \
23
+ src_c/stem_ISO_8859_1_german.c \
24
+ src_c/stem_UTF_8_german.c \
25
+ src_c/stem_ISO_8859_1_hungarian.c \
26
+ src_c/stem_UTF_8_hungarian.c \
27
+ src_c/stem_ISO_8859_1_italian.c \
28
+ src_c/stem_UTF_8_italian.c \
29
+ src_c/stem_ISO_8859_1_norwegian.c \
30
+ src_c/stem_UTF_8_norwegian.c \
31
+ src_c/stem_ISO_8859_1_porter.c \
32
+ src_c/stem_UTF_8_porter.c \
33
+ src_c/stem_ISO_8859_1_portuguese.c \
34
+ src_c/stem_UTF_8_portuguese.c \
35
+ src_c/stem_ISO_8859_2_romanian.c \
36
+ src_c/stem_UTF_8_romanian.c \
37
+ src_c/stem_KOI8_R_russian.c \
38
+ src_c/stem_UTF_8_russian.c \
39
+ src_c/stem_ISO_8859_1_spanish.c \
40
+ src_c/stem_UTF_8_spanish.c \
41
+ src_c/stem_ISO_8859_1_swedish.c \
42
+ src_c/stem_UTF_8_swedish.c \
43
+ src_c/stem_UTF_8_turkish.c \
44
+ runtime/api.c \
45
+ runtime/utilities.c \
46
+ libstemmer/libstemmer.c
47
+
48
+ snowball_headers= \
49
+ src_c/stem_ISO_8859_1_latin.h \
50
+ src_c/stem_UTF_8_latin.h \
51
+ src_c/stem_ISO_8859_1_danish.h \
52
+ src_c/stem_UTF_8_danish.h \
53
+ src_c/stem_ISO_8859_1_dutch.h \
54
+ src_c/stem_UTF_8_dutch.h \
55
+ src_c/stem_ISO_8859_1_english.h \
56
+ src_c/stem_UTF_8_english.h \
57
+ src_c/stem_ISO_8859_1_finnish.h \
58
+ src_c/stem_UTF_8_finnish.h \
59
+ src_c/stem_ISO_8859_1_french.h \
60
+ src_c/stem_UTF_8_french.h \
61
+ src_c/stem_ISO_8859_1_german.h \
62
+ src_c/stem_UTF_8_german.h \
63
+ src_c/stem_ISO_8859_1_hungarian.h \
64
+ src_c/stem_UTF_8_hungarian.h \
65
+ src_c/stem_ISO_8859_1_italian.h \
66
+ src_c/stem_UTF_8_italian.h \
67
+ src_c/stem_ISO_8859_1_norwegian.h \
68
+ src_c/stem_UTF_8_norwegian.h \
69
+ src_c/stem_ISO_8859_1_porter.h \
70
+ src_c/stem_UTF_8_porter.h \
71
+ src_c/stem_ISO_8859_1_portuguese.h \
72
+ src_c/stem_UTF_8_portuguese.h \
73
+ src_c/stem_ISO_8859_2_romanian.h \
74
+ src_c/stem_UTF_8_romanian.h \
75
+ src_c/stem_KOI8_R_russian.h \
76
+ src_c/stem_UTF_8_russian.h \
77
+ src_c/stem_ISO_8859_1_spanish.h \
78
+ src_c/stem_UTF_8_spanish.h \
79
+ src_c/stem_ISO_8859_1_swedish.h \
80
+ src_c/stem_UTF_8_swedish.h \
81
+ src_c/stem_UTF_8_turkish.h \
82
+ include/libstemmer.h \
83
+ libstemmer/modules.h \
84
+ runtime/api.h \
85
+ runtime/header.h
86
+
@@ -0,0 +1,54 @@
1
+ # libstemmer/mkinc_utf8.mak: List of stemming module source files
2
+ #
3
+ # This file is generated by mkmodules.pl from a list of module names.
4
+ # Do not edit manually.
5
+ #
6
+ # Modules included by this file are: danish, dutch, english, finnish, french,
7
+ # german, hungarian, italian, norwegian, porter, portuguese, romanian,
8
+ # russian, spanish, swedish, turkish
9
+
10
+ snowball_sources= \
11
+ src_c/stem_UTF_8_latin.c \
12
+ src_c/stem_UTF_8_danish.c \
13
+ src_c/stem_UTF_8_dutch.c \
14
+ src_c/stem_UTF_8_english.c \
15
+ src_c/stem_UTF_8_finnish.c \
16
+ src_c/stem_UTF_8_french.c \
17
+ src_c/stem_UTF_8_german.c \
18
+ src_c/stem_UTF_8_hungarian.c \
19
+ src_c/stem_UTF_8_italian.c \
20
+ src_c/stem_UTF_8_norwegian.c \
21
+ src_c/stem_UTF_8_porter.c \
22
+ src_c/stem_UTF_8_portuguese.c \
23
+ src_c/stem_UTF_8_romanian.c \
24
+ src_c/stem_UTF_8_russian.c \
25
+ src_c/stem_UTF_8_spanish.c \
26
+ src_c/stem_UTF_8_swedish.c \
27
+ src_c/stem_UTF_8_turkish.c \
28
+ runtime/api.c \
29
+ runtime/utilities.c \
30
+ libstemmer/libstemmer_utf8.c
31
+
32
+ snowball_headers= \
33
+ src_c/stem_UTF_8_latin.h \
34
+ src_c/stem_UTF_8_danish.h \
35
+ src_c/stem_UTF_8_dutch.h \
36
+ src_c/stem_UTF_8_english.h \
37
+ src_c/stem_UTF_8_finnish.h \
38
+ src_c/stem_UTF_8_french.h \
39
+ src_c/stem_UTF_8_german.h \
40
+ src_c/stem_UTF_8_hungarian.h \
41
+ src_c/stem_UTF_8_italian.h \
42
+ src_c/stem_UTF_8_norwegian.h \
43
+ src_c/stem_UTF_8_porter.h \
44
+ src_c/stem_UTF_8_portuguese.h \
45
+ src_c/stem_UTF_8_romanian.h \
46
+ src_c/stem_UTF_8_russian.h \
47
+ src_c/stem_UTF_8_spanish.h \
48
+ src_c/stem_UTF_8_swedish.h \
49
+ src_c/stem_UTF_8_turkish.h \
50
+ include/libstemmer.h \
51
+ libstemmer/modules_utf8.h \
52
+ runtime/api.h \
53
+ runtime/header.h
54
+
@@ -0,0 +1,66 @@
1
+
2
+ #include <stdlib.h> /* for calloc, free */
3
+ #include "header.h"
4
+
5
+ extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size)
6
+ {
7
+ struct SN_env * z = (struct SN_env *) calloc(1, sizeof(struct SN_env));
8
+ if (z == NULL) return NULL;
9
+ z->p = create_s();
10
+ if (z->p == NULL) goto error;
11
+ if (S_size)
12
+ {
13
+ int i;
14
+ z->S = (symbol * *) calloc(S_size, sizeof(symbol *));
15
+ if (z->S == NULL) goto error;
16
+
17
+ for (i = 0; i < S_size; i++)
18
+ {
19
+ z->S[i] = create_s();
20
+ if (z->S[i] == NULL) goto error;
21
+ }
22
+ }
23
+
24
+ if (I_size)
25
+ {
26
+ z->I = (int *) calloc(I_size, sizeof(int));
27
+ if (z->I == NULL) goto error;
28
+ }
29
+
30
+ if (B_size)
31
+ {
32
+ z->B = (unsigned char *) calloc(B_size, sizeof(unsigned char));
33
+ if (z->B == NULL) goto error;
34
+ }
35
+
36
+ return z;
37
+ error:
38
+ SN_close_env(z, S_size);
39
+ return NULL;
40
+ }
41
+
42
+ extern void SN_close_env(struct SN_env * z, int S_size)
43
+ {
44
+ if (z == NULL) return;
45
+ if (S_size)
46
+ {
47
+ int i;
48
+ for (i = 0; i < S_size; i++)
49
+ {
50
+ lose_s(z->S[i]);
51
+ }
52
+ free(z->S);
53
+ }
54
+ free(z->I);
55
+ free(z->B);
56
+ if (z->p) lose_s(z->p);
57
+ free(z);
58
+ }
59
+
60
+ extern int SN_set_current(struct SN_env * z, int size, const symbol * s)
61
+ {
62
+ int err = replace_s(z, 0, z->l, size, s, NULL);
63
+ z->c = 0;
64
+ return err;
65
+ }
66
+
@@ -0,0 +1,26 @@
1
+
2
+ typedef unsigned char symbol;
3
+
4
+ /* Or replace 'char' above with 'short' for 16 bit characters.
5
+
6
+ More precisely, replace 'char' with whatever type guarantees the
7
+ character width you need. Note however that sizeof(symbol) should divide
8
+ HEAD, defined in header.h as 2*sizeof(int), without remainder, otherwise
9
+ there is an alignment problem. In the unlikely event of a problem here,
10
+ consult Martin Porter.
11
+
12
+ */
13
+
14
+ struct SN_env {
15
+ symbol * p;
16
+ int c; int l; int lb; int bra; int ket;
17
+ symbol * * S;
18
+ int * I;
19
+ unsigned char * B;
20
+ };
21
+
22
+ extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size);
23
+ extern void SN_close_env(struct SN_env * z, int S_size);
24
+
25
+ extern int SN_set_current(struct SN_env * z, int size, const symbol * s);
26
+
@@ -0,0 +1,58 @@
1
+
2
+ #include <limits.h>
3
+
4
+ #include "api.h"
5
+
6
+ #define MAXINT INT_MAX
7
+ #define MININT INT_MIN
8
+
9
+ #define HEAD 2*sizeof(int)
10
+
11
+ #define SIZE(p) ((int *)(p))[-1]
12
+ #define SET_SIZE(p, n) ((int *)(p))[-1] = n
13
+ #define CAPACITY(p) ((int *)(p))[-2]
14
+
15
+ struct among
16
+ { int s_size; /* number of chars in string */
17
+ const symbol * s; /* search string */
18
+ int substring_i;/* index to longest matching substring */
19
+ int result; /* result of the lookup */
20
+ int (* function)(struct SN_env *);
21
+ };
22
+
23
+ extern symbol * create_s(void);
24
+ extern void lose_s(symbol * p);
25
+
26
+ extern int skip_utf8(const symbol * p, int c, int lb, int l, int n);
27
+
28
+ extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
29
+ extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
30
+ extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
31
+ extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
32
+
33
+ extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
34
+ extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
35
+ extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
36
+ extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
37
+
38
+ extern int eq_s(struct SN_env * z, int s_size, const symbol * s);
39
+ extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s);
40
+ extern int eq_v(struct SN_env * z, const symbol * p);
41
+ extern int eq_v_b(struct SN_env * z, const symbol * p);
42
+
43
+ extern int find_among(struct SN_env * z, const struct among * v, int v_size);
44
+ extern int find_among_b(struct SN_env * z, const struct among * v, int v_size);
45
+
46
+ extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjustment);
47
+ extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s);
48
+ extern int slice_from_v(struct SN_env * z, const symbol * p);
49
+ extern int slice_del(struct SN_env * z);
50
+
51
+ extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s);
52
+ extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p);
53
+
54
+ extern symbol * slice_to(struct SN_env * z, symbol * p);
55
+ extern symbol * assign_to(struct SN_env * z, symbol * p);
56
+
57
+ extern void debug(struct SN_env * z, int number, int line_count);
58
+
@@ -0,0 +1,478 @@
1
+
2
+ #include <stdio.h>
3
+ #include <stdlib.h>
4
+ #include <string.h>
5
+
6
+ #include "header.h"
7
+
8
+ #define unless(C) if(!(C))
9
+
10
+ #define CREATE_SIZE 1
11
+
12
+ extern symbol * create_s(void) {
13
+ symbol * p;
14
+ void * mem = malloc(HEAD + (CREATE_SIZE + 1) * sizeof(symbol));
15
+ if (mem == NULL) return NULL;
16
+ p = (symbol *) (HEAD + (char *) mem);
17
+ CAPACITY(p) = CREATE_SIZE;
18
+ SET_SIZE(p, CREATE_SIZE);
19
+ return p;
20
+ }
21
+
22
+ extern void lose_s(symbol * p) {
23
+ if (p == NULL) return;
24
+ free((char *) p - HEAD);
25
+ }
26
+
27
+ /*
28
+ new_p = skip_utf8(p, c, lb, l, n); skips n characters forwards from p + c
29
+ if n +ve, or n characters backwards from p + c - 1 if n -ve. new_p is the new
30
+ position, or 0 on failure.
31
+
32
+ -- used to implement hop and next in the utf8 case.
33
+ */
34
+
35
+ extern int skip_utf8(const symbol * p, int c, int lb, int l, int n) {
36
+ int b;
37
+ if (n >= 0) {
38
+ for (; n > 0; n--) {
39
+ if (c >= l) return -1;
40
+ b = p[c++];
41
+ if (b >= 0xC0) { /* 1100 0000 */
42
+ while (c < l) {
43
+ b = p[c];
44
+ if (b >= 0xC0 || b < 0x80) break;
45
+ /* break unless b is 10------ */
46
+ c++;
47
+ }
48
+ }
49
+ }
50
+ } else {
51
+ for (; n < 0; n++) {
52
+ if (c <= lb) return -1;
53
+ b = p[--c];
54
+ if (b >= 0x80) { /* 1000 0000 */
55
+ while (c > lb) {
56
+ b = p[c];
57
+ if (b >= 0xC0) break; /* 1100 0000 */
58
+ c--;
59
+ }
60
+ }
61
+ }
62
+ }
63
+ return c;
64
+ }
65
+
66
+ /* Code for character groupings: utf8 cases */
67
+
68
+ static int get_utf8(const symbol * p, int c, int l, int * slot) {
69
+ int b0, b1;
70
+ if (c >= l) return 0;
71
+ b0 = p[c++];
72
+ if (b0 < 0xC0 || c == l) { /* 1100 0000 */
73
+ * slot = b0; return 1;
74
+ }
75
+ b1 = p[c++];
76
+ if (b0 < 0xE0 || c == l) { /* 1110 0000 */
77
+ * slot = (b0 & 0x1F) << 6 | (b1 & 0x3F); return 2;
78
+ }
79
+ * slot = (b0 & 0xF) << 12 | (b1 & 0x3F) << 6 | (p[c] & 0x3F); return 3;
80
+ }
81
+
82
+ static int get_b_utf8(const symbol * p, int c, int lb, int * slot) {
83
+ int b0, b1;
84
+ if (c <= lb) return 0;
85
+ b0 = p[--c];
86
+ if (b0 < 0x80 || c == lb) { /* 1000 0000 */
87
+ * slot = b0; return 1;
88
+ }
89
+ b1 = p[--c];
90
+ if (b1 >= 0xC0 || c == lb) { /* 1100 0000 */
91
+ * slot = (b1 & 0x1F) << 6 | (b0 & 0x3F); return 2;
92
+ }
93
+ * slot = (p[c] & 0xF) << 12 | (b1 & 0x3F) << 6 | (b0 & 0x3F); return 3;
94
+ }
95
+
96
+ extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
97
+ do {
98
+ int ch;
99
+ int w = get_utf8(z->p, z->c, z->l, & ch);
100
+ unless (w) return -1;
101
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
102
+ return w;
103
+ z->c += w;
104
+ } while (repeat);
105
+ return 0;
106
+ }
107
+
108
+ extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
109
+ do {
110
+ int ch;
111
+ int w = get_b_utf8(z->p, z->c, z->lb, & ch);
112
+ unless (w) return -1;
113
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
114
+ return w;
115
+ z->c -= w;
116
+ } while (repeat);
117
+ return 0;
118
+ }
119
+
120
+ extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
121
+ do {
122
+ int ch;
123
+ int w = get_utf8(z->p, z->c, z->l, & ch);
124
+ unless (w) return -1;
125
+ unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
126
+ return w;
127
+ z->c += w;
128
+ } while (repeat);
129
+ return 0;
130
+ }
131
+
132
+ extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
133
+ do {
134
+ int ch;
135
+ int w = get_b_utf8(z->p, z->c, z->lb, & ch);
136
+ unless (w) return -1;
137
+ unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
138
+ return w;
139
+ z->c -= w;
140
+ } while (repeat);
141
+ return 0;
142
+ }
143
+
144
+ /* Code for character groupings: non-utf8 cases */
145
+
146
+ extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
147
+ do {
148
+ int ch;
149
+ if (z->c >= z->l) return -1;
150
+ ch = z->p[z->c];
151
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
152
+ return 1;
153
+ z->c++;
154
+ } while (repeat);
155
+ return 0;
156
+ }
157
+
158
+ extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
159
+ do {
160
+ int ch;
161
+ if (z->c <= z->lb) return -1;
162
+ ch = z->p[z->c - 1];
163
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
164
+ return 1;
165
+ z->c--;
166
+ } while (repeat);
167
+ return 0;
168
+ }
169
+
170
+ extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
171
+ do {
172
+ int ch;
173
+ if (z->c >= z->l) return -1;
174
+ ch = z->p[z->c];
175
+ unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
176
+ return 1;
177
+ z->c++;
178
+ } while (repeat);
179
+ return 0;
180
+ }
181
+
182
+ extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
183
+ do {
184
+ int ch;
185
+ if (z->c <= z->lb) return -1;
186
+ ch = z->p[z->c - 1];
187
+ unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
188
+ return 1;
189
+ z->c--;
190
+ } while (repeat);
191
+ return 0;
192
+ }
193
+
194
+ extern int eq_s(struct SN_env * z, int s_size, const symbol * s) {
195
+ if (z->l - z->c < s_size || memcmp(z->p + z->c, s, s_size * sizeof(symbol)) != 0) return 0;
196
+ z->c += s_size; return 1;
197
+ }
198
+
199
+ extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s) {
200
+ if (z->c - z->lb < s_size || memcmp(z->p + z->c - s_size, s, s_size * sizeof(symbol)) != 0) return 0;
201
+ z->c -= s_size; return 1;
202
+ }
203
+
204
+ extern int eq_v(struct SN_env * z, const symbol * p) {
205
+ return eq_s(z, SIZE(p), p);
206
+ }
207
+
208
+ extern int eq_v_b(struct SN_env * z, const symbol * p) {
209
+ return eq_s_b(z, SIZE(p), p);
210
+ }
211
+
212
+ extern int find_among(struct SN_env * z, const struct among * v, int v_size) {
213
+
214
+ int i = 0;
215
+ int j = v_size;
216
+
217
+ int c = z->c; int l = z->l;
218
+ symbol * q = z->p + c;
219
+
220
+ const struct among * w;
221
+
222
+ int common_i = 0;
223
+ int common_j = 0;
224
+
225
+ int first_key_inspected = 0;
226
+
227
+ while(1) {
228
+ int k = i + ((j - i) >> 1);
229
+ int diff = 0;
230
+ int common = common_i < common_j ? common_i : common_j; /* smaller */
231
+ w = v + k;
232
+ {
233
+ int i2; for (i2 = common; i2 < w->s_size; i2++) {
234
+ if (c + common == l) { diff = -1; break; }
235
+ diff = q[common] - w->s[i2];
236
+ if (diff != 0) break;
237
+ common++;
238
+ }
239
+ }
240
+ if (diff < 0) { j = k; common_j = common; }
241
+ else { i = k; common_i = common; }
242
+ if (j - i <= 1) {
243
+ if (i > 0) break; /* v->s has been inspected */
244
+ if (j == i) break; /* only one item in v */
245
+
246
+ /* - but now we need to go round once more to get
247
+ v->s inspected. This looks messy, but is actually
248
+ the optimal approach. */
249
+
250
+ if (first_key_inspected) break;
251
+ first_key_inspected = 1;
252
+ }
253
+ }
254
+ while(1) {
255
+ w = v + i;
256
+ if (common_i >= w->s_size) {
257
+ z->c = c + w->s_size;
258
+ if (w->function == 0) return w->result;
259
+ {
260
+ int res = w->function(z);
261
+ z->c = c + w->s_size;
262
+ if (res) return w->result;
263
+ }
264
+ }
265
+ i = w->substring_i;
266
+ if (i < 0) return 0;
267
+ }
268
+ }
269
+
270
+ /* find_among_b is for backwards processing. Same comments apply */
271
+
272
+ extern int find_among_b(struct SN_env * z, const struct among * v, int v_size) {
273
+
274
+ int i = 0;
275
+ int j = v_size;
276
+
277
+ int c = z->c; int lb = z->lb;
278
+ symbol * q = z->p + c - 1;
279
+
280
+ const struct among * w;
281
+
282
+ int common_i = 0;
283
+ int common_j = 0;
284
+
285
+ int first_key_inspected = 0;
286
+
287
+ while(1) {
288
+ int k = i + ((j - i) >> 1);
289
+ int diff = 0;
290
+ int common = common_i < common_j ? common_i : common_j;
291
+ w = v + k;
292
+ {
293
+ int i2; for (i2 = w->s_size - 1 - common; i2 >= 0; i2--) {
294
+ if (c - common == lb) { diff = -1; break; }
295
+ diff = q[- common] - w->s[i2];
296
+ if (diff != 0) break;
297
+ common++;
298
+ }
299
+ }
300
+ if (diff < 0) { j = k; common_j = common; }
301
+ else { i = k; common_i = common; }
302
+ if (j - i <= 1) {
303
+ if (i > 0) break;
304
+ if (j == i) break;
305
+ if (first_key_inspected) break;
306
+ first_key_inspected = 1;
307
+ }
308
+ }
309
+ while(1) {
310
+ w = v + i;
311
+ if (common_i >= w->s_size) {
312
+ z->c = c - w->s_size;
313
+ if (w->function == 0) return w->result;
314
+ {
315
+ int res = w->function(z);
316
+ z->c = c - w->s_size;
317
+ if (res) return w->result;
318
+ }
319
+ }
320
+ i = w->substring_i;
321
+ if (i < 0) return 0;
322
+ }
323
+ }
324
+
325
+
326
+ /* Increase the size of the buffer pointed to by p to at least n symbols.
327
+ * If insufficient memory, returns NULL and frees the old buffer.
328
+ */
329
+ static symbol * increase_size(symbol * p, int n) {
330
+ symbol * q;
331
+ int new_size = n + 20;
332
+ void * mem = realloc((char *) p - HEAD,
333
+ HEAD + (new_size + 1) * sizeof(symbol));
334
+ if (mem == NULL) {
335
+ lose_s(p);
336
+ return NULL;
337
+ }
338
+ q = (symbol *) (HEAD + (char *)mem);
339
+ CAPACITY(q) = new_size;
340
+ return q;
341
+ }
342
+
343
+ /* to replace symbols between c_bra and c_ket in z->p by the
344
+ s_size symbols at s.
345
+ Returns 0 on success, -1 on error.
346
+ Also, frees z->p (and sets it to NULL) on error.
347
+ */
348
+ extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjptr)
349
+ {
350
+ int adjustment;
351
+ int len;
352
+ if (z->p == NULL) {
353
+ z->p = create_s();
354
+ if (z->p == NULL) return -1;
355
+ }
356
+ adjustment = s_size - (c_ket - c_bra);
357
+ len = SIZE(z->p);
358
+ if (adjustment != 0) {
359
+ if (adjustment + len > CAPACITY(z->p)) {
360
+ z->p = increase_size(z->p, adjustment + len);
361
+ if (z->p == NULL) return -1;
362
+ }
363
+ memmove(z->p + c_ket + adjustment,
364
+ z->p + c_ket,
365
+ (len - c_ket) * sizeof(symbol));
366
+ SET_SIZE(z->p, adjustment + len);
367
+ z->l += adjustment;
368
+ if (z->c >= c_ket)
369
+ z->c += adjustment;
370
+ else
371
+ if (z->c > c_bra)
372
+ z->c = c_bra;
373
+ }
374
+ unless (s_size == 0) memmove(z->p + c_bra, s, s_size * sizeof(symbol));
375
+ if (adjptr != NULL)
376
+ *adjptr = adjustment;
377
+ return 0;
378
+ }
379
+
380
+ static int slice_check(struct SN_env * z) {
381
+
382
+ if (z->bra < 0 ||
383
+ z->bra > z->ket ||
384
+ z->ket > z->l ||
385
+ z->p == NULL ||
386
+ z->l > SIZE(z->p)) /* this line could be removed */
387
+ {
388
+ #if 0
389
+ fprintf(stderr, "faulty slice operation:\n");
390
+ debug(z, -1, 0);
391
+ #endif
392
+ return -1;
393
+ }
394
+ return 0;
395
+ }
396
+
397
+ extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s) {
398
+ if (slice_check(z)) return -1;
399
+ return replace_s(z, z->bra, z->ket, s_size, s, NULL);
400
+ }
401
+
402
+ extern int slice_from_v(struct SN_env * z, const symbol * p) {
403
+ return slice_from_s(z, SIZE(p), p);
404
+ }
405
+
406
+ extern int slice_del(struct SN_env * z) {
407
+ return slice_from_s(z, 0, 0);
408
+ }
409
+
410
+ extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s) {
411
+ int adjustment;
412
+ if (replace_s(z, bra, ket, s_size, s, &adjustment))
413
+ return -1;
414
+ if (bra <= z->bra) z->bra += adjustment;
415
+ if (bra <= z->ket) z->ket += adjustment;
416
+ return 0;
417
+ }
418
+
419
+ extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p) {
420
+ int adjustment;
421
+ if (replace_s(z, bra, ket, SIZE(p), p, &adjustment))
422
+ return -1;
423
+ if (bra <= z->bra) z->bra += adjustment;
424
+ if (bra <= z->ket) z->ket += adjustment;
425
+ return 0;
426
+ }
427
+
428
+ extern symbol * slice_to(struct SN_env * z, symbol * p) {
429
+ if (slice_check(z)) {
430
+ lose_s(p);
431
+ return NULL;
432
+ }
433
+ {
434
+ int len = z->ket - z->bra;
435
+ if (CAPACITY(p) < len) {
436
+ p = increase_size(p, len);
437
+ if (p == NULL)
438
+ return NULL;
439
+ }
440
+ memmove(p, z->p + z->bra, len * sizeof(symbol));
441
+ SET_SIZE(p, len);
442
+ }
443
+ return p;
444
+ }
445
+
446
+ extern symbol * assign_to(struct SN_env * z, symbol * p) {
447
+ int len = z->l;
448
+ if (CAPACITY(p) < len) {
449
+ p = increase_size(p, len);
450
+ if (p == NULL)
451
+ return NULL;
452
+ }
453
+ memmove(p, z->p, len * sizeof(symbol));
454
+ SET_SIZE(p, len);
455
+ return p;
456
+ }
457
+
458
+ #if 0
459
+ extern void debug(struct SN_env * z, int number, int line_count) {
460
+ int i;
461
+ int limit = SIZE(z->p);
462
+ /*if (number >= 0) printf("%3d (line %4d): '", number, line_count);*/
463
+ if (number >= 0) printf("%3d (line %4d): [%d]'", number, line_count,limit);
464
+ for (i = 0; i <= limit; i++) {
465
+ if (z->lb == i) printf("{");
466
+ if (z->bra == i) printf("[");
467
+ if (z->c == i) printf("|");
468
+ if (z->ket == i) printf("]");
469
+ if (z->l == i) printf("}");
470
+ if (i < limit)
471
+ { int ch = z->p[i];
472
+ if (ch == 0) ch = '#';
473
+ printf("%c", ch);
474
+ }
475
+ }
476
+ printf("'\n");
477
+ }
478
+ #endif