ruby_wordcram 1.0.1 → 2.0.0

data/src/cue/lang/NGramIterator.java

@@ -0,0 +1,151 @@
+/*
+   Copyright 2009 IBM Corp
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+ */
+package cue.lang;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Locale;
+import java.util.NoSuchElementException;
+
+import cue.lang.stop.StopWords;
+
+/**
+ * Construct with a {@link String}, some integer n, and a {@link Locale};
+ * retrieve a sequence of {@link String}s, each of which has n words that appear
+ * contiguously within a sentence. "Words" are as defined by the
+ * {@link WordIterator}.
+ * 
+ * <p>
+ * If you don't provide a {@link Locale}, you get the default {@link Locale} for
+ * your system, which may or may not be what you want. The {@link Locale} is
+ * used by a {@link SentenceIterator} to find sentence breaks.
+ * 
+ * <p>
+ * Example:
+ * 
+ * <pre>
+ * final String lyric = "This happened once before. I came to your door. No reply.";
+ * for (final String s : new NGramIterator(3, lyric)) {
+ *     System.out.println(s);
+ * }
+ * for (final String s : new NGramIterator(2, lyric)) {
+ *     System.out.println(s);
+ * }
+ *  
+ * This happened once
+ * happened once before
+ * I came to
+ * came to your
+ * to your door
+ * 
+ * This happened
+ * happened once
+ * once before
+ * I came
+ * came to
+ * to your
+ * your door
+ * No reply
+ * </pre>
+ * 
+ * @author Jonathan Feinberg <jdf@us.ibm.com>
+ * 
+ */
+public class NGramIterator extends IterableText {
+	private final SentenceIterator sentenceIterator;
+	private final LinkedList<String> grams = new LinkedList<String>();
+	private final int n;
+	private final StopWords stopWords;
+
+	private String next;
+	private Iterator<String> currentWordIterator;
+
+	public NGramIterator(final int n, final String text) {
+		this(n, text, Locale.getDefault());
+	}
+
+	public NGramIterator(final int n, final String text, final Locale locale) {
+		this(n, text, locale, null);
+	}
+
+	public NGramIterator(final int n, final String text, final Locale locale,
+			final StopWords stopWords) {
+		this.n = n;
+		this.sentenceIterator = new SentenceIterator(text, locale);
+		this.stopWords = stopWords;
+		loadNext();
+	}
+
+        @Override
+	public void remove() {
+		throw new UnsupportedOperationException();
+	}
+
+        @Override
+	public String next() {
+		if (next == null) {
+			throw new NoSuchElementException();
+		}
+		final String result = next;
+		loadNext();
+		return result;
+	}
+
+        @Override
+	public boolean hasNext() {
+		return next != null;
+	}
+
+	private void loadNext() {
+		next = null;
+		if (!grams.isEmpty()) {
+			grams.pop();
+		}
+		while (grams.size() < n) {
+			while (currentWordIterator == null
+					|| !currentWordIterator.hasNext()) {
+				if (!sentenceIterator.hasNext()) {
+					return;
+				}
+				grams.clear();
+				currentWordIterator = new WordIterator(sentenceIterator.next())
+						.iterator();
+				for (int i = 0; currentWordIterator.hasNext() && i < n - 1; i++) {
+					maybeAddWord();
+				}
+			}
+			// now grams has n-1 words in it and currentWordIterator hasNext
+			maybeAddWord();
+		}
+		final StringBuilder sb = new StringBuilder();
+		for (final String gram : grams) {
+			if (sb.length() > 0) {
+				sb.append(" ");
+			}
+			sb.append(gram);
+		}
+		next = sb.toString();
+	}
+
+	private void maybeAddWord() {
+		final String nextWord = currentWordIterator.next();
+		if (stopWords != null && stopWords.isStopWord(nextWord)) {
+			grams.clear();
+		} else {
+			grams.add(nextWord);
+		}
+	}
+}

data/src/cue/lang/SentenceIterator.java

@@ -0,0 +1,86 @@
+/*
+   Copyright 2009 IBM Corp
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+ */
+package cue.lang;
+
+import java.text.BreakIterator;
+import java.util.Locale;
+import java.util.NoSuchElementException;
+import java.util.regex.Pattern;
+
+/**
+ * Construct with a {@link String}; retrieve a sequence of {@link String}s, each
+ * of which is a "sentence" according to Java's built-in model for the given
+ * {@link Locale}.
+ * 
+ * @author Jonathan Feinberg <jdf@us.ibm.com>
+ * 
+ */
+public class SentenceIterator extends IterableText {
+	private final String text;
+	private final BreakIterator breakIterator;
+	int start, end;
+
+	/**
+	 * Uses the default {@link Locale} for the running instance of the JVM.
+	 * 
+	 * @param text
+	 */
+	public SentenceIterator(final String text) {
+		this(text, Locale.getDefault());
+	}
+
+	public SentenceIterator(final String text, final Locale locale) {
+		this.text = text;
+		breakIterator = BreakIterator.getSentenceInstance(locale);
+		breakIterator.setText(text);
+		start = end = breakIterator.first();
+		advance();
+	}
+
+	private static final Pattern ABBREVS = Pattern
+			.compile("(?:Mrs?|Ms|Dr|Rev)\\.\\s*$");
+
+	private void advance() {
+		start = end;
+		while (hasNext()
+				&& ((end == start) || ABBREVS.matcher(
+						text.substring(start, end)).find())) {
+			end = breakIterator.next();
+		}
+	}
+
+        @Override
+	public void remove() {
+		throw new UnsupportedOperationException();
+	}
+
+        @Override
+	public String next() {
+		if (!hasNext()) {
+			throw new NoSuchElementException();
+		}
+		final String result = text.substring(start, end)
+				.replaceAll("\\s+", " ");
+		advance();
+		return result;
+	}
+
+        @Override
+	public final boolean hasNext() {
+		return end != BreakIterator.DONE;
+	}
+
+}

data/src/cue/lang/WordIterator.java

@@ -0,0 +1,60 @@
+/*
+   Copyright 2009 IBM Corp
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+ */
+package cue.lang;
+
+import java.util.NoSuchElementException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * 
+ * @author Jonathan Feinberg <jdf@us.ibm.com>
+ * 
+ */
+public class WordIterator extends IterableText {
+    private static final String LETTER = "[@+\\p{javaLetterOrDigit}]";
+    private static final String JOINER = "[-.:/'’\\p{M}\\u2032\\u00A0\\u200C\\u200D~]";
+    private static final Pattern WORD = Pattern.compile(LETTER + "+(" + JOINER + "+" + LETTER
+            + "+)*");
+
+    private final Matcher m;
+    private boolean hasNext;
+
+    public WordIterator(final String text) {
+        this.m = WORD.matcher(text == null ? "" : text);
+        hasNext = m.find();
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public String next() {
+        if (!hasNext) {
+            throw new NoSuchElementException();
+        }
+        final String s = m.group();
+        hasNext = m.find();
+        return s;
+    }
+
+    @Override
+    public boolean hasNext() {
+        return hasNext;
+    }
+}

data/src/cue/lang/stop/StopWords.java

@@ -0,0 +1,114 @@
+/*
+   Copyright 2009 IBM Corp
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+ */
+
+package cue.lang.stop;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Set;
+
+import cue.lang.Counter;
+import cue.lang.WordIterator;
+
+/**
+ * 
+ * @author Jonathan Feinberg <jdf@us.ibm.com>
+ * 
+ */
+public enum StopWords {
+    Arabic(), Armenian(), Catalan(true), Croatian(), Czech(), Dutch(), //
+    Danish(), English(), Esperanto(), Farsi(), Finnish(), //
+    French(true), German(), Greek(), Hindi(), Hungarian(), //
+    Italian(), Latin(), Norwegian(), Polish(), Portuguese(), //
+    Romanian(), Russian(), Slovenian(), Slovak(), Spanish(), //
+    Swedish(), Hebrew(), Turkish(), Custom();
+
+    public static StopWords guess(final String text) {
+        return guess(new Counter<>(new WordIterator(text)));
+    }
+
+    public static StopWords guess(final Counter<String> wordCounter) {
+        return guess(wordCounter.getMostFrequent(50));
+    }
+
+    public static StopWords guess(final Collection<String> words) {
+        StopWords currentWinner = null;
+        int currentMax = 0;
+        for (final StopWords stopWords : StopWords.values()) {
+            int count = 0;
+            count = words.stream().filter((word) -> (stopWords.isStopWord(word))).map((_item) -> 1).reduce(count, Integer::sum);
+            if (count > currentMax) {
+                currentWinner = stopWords;
+                currentMax = count;
+            }
+        }
+        return currentWinner;
+    }
+
+    public final boolean stripApostrophes;
+    private final Set<String> stopwords = new HashSet<>();
+
+    private StopWords() {
+        this(false);
+    }
+
+    private StopWords(final boolean stripApostrophes) {
+        this.stripApostrophes = stripApostrophes;
+        loadLanguage();
+    }
+
+    public boolean isStopWord(final String s) {
+        if (s.length() == 1) {
+            return true;
+        }
+        // check rightquotes as apostrophes
+        return stopwords.contains(s.replace('\u2019', '\'').toLowerCase(Locale.ENGLISH));
+    }
+
+    private void loadLanguage() {
+        final String wordlistResource = name().toLowerCase(Locale.ENGLISH);
+        if (!wordlistResource.equals("custom")) {
+            readStopWords(getClass().getResourceAsStream(wordlistResource),
+                    Charset.forName("UTF-8"));
+        }
+    }
+
+    public void readStopWords(final InputStream inputStream, final Charset encoding) {
+        try {
+            try (BufferedReader in = new BufferedReader(new InputStreamReader(inputStream,
+                    encoding))) {
+                String line;
+                while ((line = in.readLine()) != null) {
+                    line = line.replaceAll("\\|.*", "").trim();
+                    if (line.length() == 0) {
+                        continue;
+                    }
+                    for (final String w : line.split("\\s+")) {
+                        stopwords.add(w.toLowerCase(Locale.ENGLISH));
+                    }
+                }
+            }
+        } catch (final IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

data/src/cue/lang/stop/arabic

@@ -0,0 +1,351 @@
+إلى
+أو
+إننا
+إن
+أن
+ب
+ا
+أ
+،
+عشر
+عبد
+عدد
+عدة
+عشرة
+عدم
+عام
+عاما
+عرفات
+عن
+عند
+عمان
+عندما
+على
+علي
+عليه
+عليها
+عملية
+زيارة
+سبتمبر
+ساراييفو
+سنة
+سوريا
+سنوات
+تشرين
+تم
+تموز
+ضد
+بعد
+بعض
+اعادة
+اعلن
+اعلنت
+حزب
+حزيران
+بسبب
+اسرائيل
+حسين
+حتى
+اتفاق
+صرب
+اذا
+احد
+اثر
+غزة
+برس
+باسم
+اجتماع
+غدا
+شخصا
+صباح
+اطار
+اربعة
+بغداد
+اخرى
+باريس
+رابين
+شرق
+بان
+ابو
+اجل
+غير
+حركة
+رئيس
+جديدة
+اطلاق
+بشكل
+بطولة
+صحيفة
+حاليا
+بن
+به
+ثم
+اف
+ان
+او
+اي
+بها
+جهة
+صفر
+حيث
+اكد
+الا
+اما
+العسكرية
+العراق
+العاصمة
+العربية
+العراقي
+العراقية
+العام
+العالم
+العلاقات
+العمل
+امس
+السعودية
+الساعة
+السبت
+السابق
+روسيا
+السلطة
+السلطات
+السلام
+التعاون
+التحرير
+التى
+التي
+اكتوبر
+دورة
+اكثر
+ايار
+ايضا
+الجزائر
+حماس
+الاسرائيلي
+الاسرائيلية
+الاسبوع
+الاسلحة
+الاسلامية
+ذكرت
+الاتحاد
+الاتفاق
+ثلاثة
+الحرب
+الاحد
+الذاتي
+الشرطة
+الاربعاء
+الغربية
+الخارجية
+الاردن
+الشرق
+ايران
+الحدود
+الرئيس
+الاخيرة
+الثاني
+الثانية
+الاثنين
+شمال
+بيان
+دمشق
+الذى
+الذي
+الان
+امام
+ايام
+خلال
+الشيخ
+الجيش
+الدور
+الضفة
+الجمعة
+بيريز
+الاوسط
+الروسي
+البوسنة
+الروسية
+بيروت
+الانتخابات
+البلاد
+الدفاع
+الثلثاء
+الانباء
+الثلاثاء
+الاوروبي
+حوالى
+الذين
+الدول
+الحكم
+الامم
+الامن
+الاول
+الدولة
+الخليج
+الخميس
+الاميركي
+الاميركية
+الدولي
+الاولى
+الدولية
+الحكومة
+بين
+ذلك
+دول
+دون
+حول
+حين
+الف
+الى
+انه
+اول
+ضمن
+جنوب
+دولة
+انها
+جميع
+الوزراء
+المتحدث
+المتحدة
+دولار
+النار
+الوضع
+القدس
+المحتلة
+المصدر
+المباراة
+المصري
+الماضي
+المصرية
+المرحلة
+القدم
+اللجنة
+المجلس
+الفرنسي
+الفرنسية
+القاهرة
+المدينة
+المانيا
+الوطنية
+المجموعة
+الفلسطيني
+الفلسطينية
+الفلسطينيين
+الوقت
+المقرر
+القوات
+النهائي
+المقبل
+المنطقة
+الولايات
+المفاوضات
+الملك
+اليمن
+اليوم
+ايلول
+الكويت
+ـ
+ف
+و
+و6
+قد
+لا
+ما
+مع
+وزارة
+وزير
+مساء
+قتل
+كرة
+مصر
+هذا
+فاز
+كأس
+ياسر
+قرار
+مصدر
+واحد
+قطاع
+مصادر
+مباراة
+مبارك
+واضاف
+واضافت
+فرانس
+واشنطن
+فان
+قبل
+قال
+كان
+لدى
+نحو
+هذه
+وان
+محمد
+واكد
+يذكر
+مجلس
+فرنسا
+كريستوفر
+كانت
+واوضح
+لبنان
+مايو
+مدينة
+مجموعة
+كانون
+فى
+في
+كل
+لم
+لن
+له
+من
+هو
+هي
+قوة
+كما
+لها
+منذ
+وقد
+ولا
+نفسه
+موسكو
+مقتل
+لقاء
+لكرة
+نقطة
+قوات
+مقابل
+لندن
+هناك
+وقال
+وكان
+منطقة
+منظمة
+نهاية
+وكالة
+وقالت
+وكانت
+للامم
+فيه
+كلم
+لكن
+وفي
+وقف
+ولم
+ومن
+وهو
+وهي
+يوم
+فيها
+منها
+مليار
+لوكالة
+يكون
+يمكن
+كلينتون
+مليون
+يوليو
+يونيو
+نيويورك