scrapedo 0.1.0

data/lib/scrapedo/config/google_builder/host_language.yml

@@ -0,0 +1,469 @@
+common: &common_data
+  # English
+  hl_en: "en"
+  hl_english: "en"
+  # Japanese
+  hl_ja: "ja"
+  hl_japanese: "ja"
+  # Turkish
+  hl_tr: "tr"
+  hl_turkish: "tr"
+  # Korean
+  hl_ko: "ko"
+  hl_korean: "ko"
+  # German
+  hl_de: "de"
+  hl_german: "de"
+  # Arabic
+  hl_ar: "ar"
+  hl_arabic: "ar"
+  # French
+  hl_fr: "fr"
+  hl_french: "fr"
+  # Russian
+  hl_ru: "ru"
+  hl_russian: "ru"
+  # Spanish
+  hl_es: "es"
+  hl_spanish: "es"
+  # Chinese (Simplified)
+  hl_zh_cn: "zh-cn"
+  hl_chinese_simplified: "zh-cn"
+  # Chinese (Traditional)
+  hl_zh_tw: "zh-tw"
+  hl_chinese_traditional: "zh-tw"
+  # Italian
+  hl_it: "it"
+  hl_italian: "it"
+  # Portuguese
+  hl_pt: "pt"
+  hl_portuguese: "pt"
+  # Polish
+  hl_pl: "pl"
+  hl_polish: "pl"
+  # Dutch
+  hl_nl: "nl"
+  hl_dutch: "nl"
+  # Hindi
+  hl_hi: "hi"
+  hl_hindi: "hi"
+  # Swedish
+  hl_sv: "sv"
+  hl_swedish: "sv"
+  # Thai
+  hl_th: "th"
+  hl_thai: "th"
+all:
+  <<: *common_data
+  # Luo
+  hl_ach: "ach"
+  hl_luo: "ach"
+  # Afrikaans
+  hl_af: "af"
+  hl_afrikaans: "af"
+  # Akan
+  hl_ak: "ak"
+  hl_akan: "ak"
+  # Amharic
+  hl_am: "am"
+  hl_amharic: "am"
+  # Azerbaijani
+  hl_az: "az"
+  hl_azerbaijani: "az"
+  # Belarusian
+  hl_be: "be"
+  hl_belarusian: "be"
+  # Bemba
+  hl_bem: "bem"
+  hl_bemba: "bem"
+  # Bihari
+  hl_bh: "bh"
+  hl_bihari: "bh"
+  # Bengali
+  hl_bn: "bn"
+  hl_bengali: "bn"
+  # Breton
+  hl_br: "br"
+  hl_breton: "br"
+  # Bosnian
+  hl_bs: "bs"
+  hl_bosnian: "bs"
+  # Bhutanese
+  hl_bt: "bt"
+  hl_bhutanese: "bt"
+  # Bulgarian
+  hl_bg: "bg"
+  hl_bulgarian: "bg"
+  # Catalan
+  hl_ca: "ca"
+  hl_catalan: "ca"
+  # Cherokee
+  hl_chr: "chr"
+  hl_cherokee: "chr"
+  # Kurdish (Sorani)
+  hl_ckb: "ckb"
+  hl_kurdish_sorani: "ckb"
+  hl_sorani: "ckb"
+  # Corsican
+  hl_co: "co"
+  hl_corsican: "co"
+  # Seychellois Creole
+  hl_crs: "crs"
+  hl_seychellois_creole: "crs"
+  # Czech
+  hl_cs: "cs"
+  hl_czech: "cs"
+  # Welsh
+  hl_cy: "cy"
+  hl_welsh: "cy"
+  # Danish
+  hl_da: "da"
+  hl_danish: "da"
+  # Ewe
+  hl_ee: "ee"
+  hl_ewe: "ee"
+  # Greek
+  hl_el: "el"
+  hl_greek: "el"
+  # Esperanto
+  hl_eo: "eo"
+  hl_esperanto: "eo"
+  # Spanish (Latin America)
+  hl_es_419: "es-419"
+  hl_spanish_latin_america: "es-419"
+  # Estonian
+  hl_et: "et"
+  hl_estonian: "et"
+  # Basque
+  hl_eu: "eu"
+  hl_basque: "eu"
+  # Persian
+  hl_fa: "fa"
+  hl_persian: "fa"
+  # Finnish
+  hl_fi: "fi"
+  hl_finnish: "fi"
+  # Faroese
+  hl_fo: "fo"
+  hl_faroese: "fo"
+  # Frisian
+  hl_fy: "fy"
+  hl_frisian: "fy"
+  # Irish
+  hl_ga: "ga"
+  hl_irish: "ga"
+  # Ga
+  hl_gaa: "gaa"
+  hl_ga_language: "gaa"
+  # Scots Gaelic
+  hl_gd: "gd"
+  hl_scots_gaelic: "gd"
+  # Galician
+  hl_gl: "gl"
+  hl_galician: "gl"
+  # Guarani
+  hl_gn: "gn"
+  hl_guarani: "gn"
+  # Gujarati
+  hl_gu: "gu"
+  hl_gujarati: "gu"
+  # Hausa
+  hl_ha: "ha"
+  hl_hausa: "ha"
+  # Hawaiian
+  hl_haw: "haw"
+  hl_hawaiian: "haw"
+  # Hebrew
+  hl_he: "he"
+  hl_hebrew: "he"
+  # Croatian
+  hl_hr: "hr"
+  hl_croatian: "hr"
+  # Haitian Creole
+  hl_ht: "ht"
+  hl_haitian_creole: "ht"
+  # Hungarian
+  hl_hu: "hu"
+  hl_hungarian: "hu"
+  # Armenian
+  hl_hy: "hy"
+  hl_armenian: "hy"
+  # Interlingua
+  hl_ia: "ia"
+  hl_interlingua: "ia"
+  # Indonesian
+  hl_id: "id"
+  hl_indonesian: "id"
+  # Igbo
+  hl_ig: "ig"
+  hl_igbo: "ig"
+  # Icelandic
+  hl_is: "is"
+  hl_icelandic: "is"
+  # Hebrew (legacy)
+  hl_iw: "iw"
+  hl_hebrew_legacy: "iw"
+  # Javanese
+  hl_jw: "jw"
+  hl_javanese: "jw"
+  # Georgian
+  hl_ka: "ka"
+  hl_georgian: "ka"
+  # Kongo
+  hl_kg: "kg"
+  hl_kongo: "kg"
+  # Kazakh
+  hl_kk: "kk"
+  hl_kazakh: "kk"
+  # Greenlandic
+  hl_kl: "kl"
+  hl_greenlandic: "kl"
+  # Cambodian
+  hl_km: "km"
+  hl_cambodian: "km"
+  hl_khmer: "km"
+  # Kannada
+  hl_kn: "kn"
+  hl_kannada: "kn"
+  # Krio
+  hl_kri: "kri"
+  hl_krio: "kri"
+  # Kurdish
+  hl_ku: "ku"
+  hl_kurdish: "ku"
+  # Kyrgyz
+  hl_ky: "ky"
+  hl_kyrgyz: "ky"
+  # Latin
+  hl_la: "la"
+  hl_latin: "la"
+  # Luganda
+  hl_lg: "lg"
+  hl_luganda: "lg"
+  # Lingala
+  hl_ln: "ln"
+  hl_lingala: "ln"
+  # Laothian
+  hl_lo: "lo"
+  hl_laothian: "lo"
+  hl_lao: "lo"
+  # Lozi
+  hl_loz: "loz"
+  hl_lozi: "loz"
+  # Lithuanian
+  hl_lt: "lt"
+  hl_lithuanian: "lt"
+  # Tshiluba
+  hl_lua: "lua"
+  hl_tshiluba: "lua"
+  # Latvian
+  hl_lv: "lv"
+  hl_latvian: "lv"
+  # Mauritian Creole
+  hl_mfe: "mfe"
+  hl_mauritian_creole: "mfe"
+  # Malagasy
+  hl_mg: "mg"
+  hl_malagasy: "mg"
+  # Maori
+  hl_mi: "mi"
+  hl_maori: "mi"
+  # Macedonian
+  hl_mk: "mk"
+  hl_macedonian: "mk"
+  # Malayalam
+  hl_ml: "ml"
+  hl_malayalam: "ml"
+  # Mongolian
+  hl_mn: "mn"
+  hl_mongolian: "mn"
+  # Moldavian
+  hl_mo: "mo"
+  hl_moldavian: "mo"
+  # Marathi
+  hl_mr: "mr"
+  hl_marathi: "mr"
+  # Malay
+  hl_ms: "ms"
+  hl_malay: "ms"
+  # Maltese
+  hl_mt: "mt"
+  hl_maltese: "mt"
+  # Maldives
+  hl_mv: "mv"
+  hl_maldives: "mv"
+  hl_dhivehi: "mv"
+  # Myanmar (Burmese)
+  hl_my: "my"
+  hl_myanmar: "my"
+  hl_burmese: "my"
+  # Nepali
+  hl_ne: "ne"
+  hl_nepali: "ne"
+  # Norwegian (Nynorsk)
+  hl_nn: "nn"
+  hl_norwegian_nynorsk: "nn"
+  # Norwegian
+  hl_no: "no"
+  hl_norwegian: "no"
+  # Northern Sotho
+  hl_nso: "nso"
+  hl_northern_sothon: "nso"
+  # Chichewa
+  hl_ny: "ny"
+  hl_chichewa: "ny"
+  # Runyakitara
+  hl_nyn: "nyn"
+  hl_runyakitara: "nyn"
+  # Occitan
+  hl_oc: "oc"
+  hl_occitan: "oc"
+  # Oromo
+  hl_om: "om"
+  hl_oromo: "om"
+  # Oriya
+  hl_or: "or"
+  hl_oriya: "or"
+  hl_odia: "or"
+  # Punjabi
+  hl_pa: "pa"
+  hl_punjabi: "pa"
+  # Nigerian Pidgin
+  hl_pcm: "pcm"
+  hl_nigerian_pidgin: "pcm"
+  # Pashto
+  hl_ps: "ps"
+  hl_pashto: "ps"
+  # Portuguese (Brazil)
+  hl_pt_br: "pt-br"
+  hl_portuguese_brazil: "pt-br"
+  # Portuguese (Portugal)
+  hl_pt_pt: "pt-pt"
+  hl_portuguese_portugal: "pt-pt"
+  # Quechua
+  hl_qu: "qu"
+  hl_quechua: "qu"
+  # Romansh
+  hl_rm: "rm"
+  hl_romansh: "rm"
+  # Kirundi
+  hl_rn: "rn"
+  hl_kirundi: "rn"
+  # Romanian
+  hl_ro: "ro"
+  hl_romanian: "ro"
+  # Kinyarwanda
+  hl_rw: "rw"
+  hl_kinyarwanda: "rw"
+  # Sindhi
+  hl_sd: "sd"
+  hl_sindhi: "sd"
+  # Serbo-Croatian
+  hl_sh: "sh"
+  hl_serbo_croatian: "sh"
+  # Sinhalese
+  hl_si: "si"
+  hl_sinhalese: "si"
+  hl_sinhala: "si"
+  # Slovak
+  hl_sk: "sk"
+  hl_slovak: "sk"
+  # Slovenian
+  hl_sl: "sl"
+  hl_slovenian: "sl"
+  # Shona
+  hl_sn: "sn"
+  hl_shona: "sn"
+  # Somali
+  hl_so: "so"
+  hl_somali: "so"
+  # Albanian
+  hl_sq: "sq"
+  hl_albanian: "sq"
+  # Serbian
+  hl_sr: "sr"
+  hl_serbian: "sr"
+  # Montenegrin
+  hl_sr_me: "sr-me"
+  hl_montenegrin: "sr-me"
+  # Sesotho
+  hl_st: "st"
+  hl_sesotho: "st"
+  # Sundanese
+  hl_su: "su"
+  hl_sundanese: "su"
+  # Swahili
+  hl_sw: "sw"
+  hl_swahili: "sw"
+  # Tamil
+  hl_ta: "ta"
+  hl_tamil: "ta"
+  # Telugu
+  hl_te: "te"
+  hl_telugu: "te"
+  # Tajik
+  hl_tg: "tg"
+  hl_tajik: "tg"
+  # Tigrinya
+  hl_ti: "ti"
+  hl_tigrinya: "ti"
+  # Turkmen
+  hl_tk: "tk"
+  hl_turkmen: "tk"
+  # Filipino
+  hl_tl: "tl"
+  hl_filipino: "tl"
+  hl_tagalog: "tl"
+  # Setswana
+  hl_tn: "tn"
+  hl_setswana: "tn"
+  # Tonga
+  hl_to: "to"
+  hl_tonga: "to"
+  # Tatar
+  hl_tt: "tt"
+  hl_tatar: "tt"
+  # Tumbuka
+  hl_tum: "tum"
+  hl_tumbuka: "tum"
+  # Twi
+  hl_tw: "tw"
+  hl_twi: "tw"
+  # Uighur
+  hl_ug: "ug"
+  hl_uighur: "ug"
+  hl_uyghur: "ug"
+  # Ukrainian
+  hl_uk: "uk"
+  hl_ukrainian: "uk"
+  # Urdu
+  hl_ur: "ur"
+  hl_urdu: "ur"
+  # Uzbek
+  hl_uz: "uz"
+  hl_uzbek: "uz"
+  # Vietnamese
+  hl_vi: "vi"
+  hl_vietnamese: "vi"
+  # Vanuatu
+  hl_vu: "vu"
+  hl_vanuatu: "vu"
+  hl_bisalama: "vu"
+  # Wolof
+  hl_wo: "wo"
+  hl_wolof: "wo"
+  # Samoa
+  hl_ws: "ws"
+  hl_samoa: "ws"
+  # Xhosa
+  hl_xh: "xh"
+  hl_xhosa: "xh"
+  # Yiddish
+  hl_yi: "yi"
+  hl_yiddish: "yi"
+  # Yoruba
+  hl_yo: "yo"
+  hl_yoruba: "yo"
+  # Zulu
+  hl_zu: "zu"
+  hl_zulu: "zu"

data/lib/scrapedo/config/google_builder/language_restrict.yml

@@ -0,0 +1,107 @@
+common: &common_data
+  # Arabic
+  lr_ar: "lang_ar"
+  lr_arabic: "lang_ar"
+  # Hungarian
+  lr_hu: "lang_hu"
+  lr_hungarian: "lang_hu"
+  # Polish
+  lr_pl: "lang_pl"
+  lr_polish: "lang_pl"
+  # Bulgarian
+  lr_bg: "lang_bg"
+  lr_bulgarian: "lang_bg"
+  # Indonesian
+  lr_id: "lang_id"
+  lr_indonesian: "lang_id"
+  # Portuguese
+  lr_pt: "lang_pt"
+  lr_portuguese: "lang_pt"
+  # Catalan
+  lr_ca: "lang_ca"
+  lr_catalan: "lang_ca"
+  # Icelandic
+  lr_is: "lang_is"
+  lr_icelandic: "lang_is"
+  # Romanian
+  lr_ro: "lang_ro"
+  lr_romanian: "lang_ro"
+  # Czech
+  lr_cs: "lang_cs"
+  lr_czech: "lang_cs"
+  # Italian
+  lr_it: "lang_it"
+  lr_italian: "lang_it"
+  # Russian
+  lr_ru: "lang_ru"
+  lr_russian: "lang_ru"
+  # Danish
+  lr_da: "lang_da"
+  lr_danish: "lang_da"
+  # Hebrew
+  lr_iw: "lang_iw"
+  lr_hebrew: "lang_iw"
+  # Slovak
+  lr_sk: "lang_sk"
+  lr_slovak: "lang_sk"
+  # German
+  lr_de: "lang_de"
+  lr_german: "lang_de"
+  # Japanese
+  lr_ja: "lang_ja"
+  lr_japanese: "lang_ja"
+  # Slovenian
+  lr_sl: "lang_sl"
+  lr_slovenian: "lang_sl"
+  # Greek
+  lr_el: "lang_el"
+  lr_greek: "lang_el"
+  # Korean
+  lr_ko: "lang_ko"
+  lr_korean: "lang_ko"
+  # Serbian
+  lr_sr: "lang_sr"
+  lr_serbian: "lang_sr"
+  # English
+  lr_en: "lang_en"
+  lr_english: "lang_en"
+  # Lithuanian
+  lr_lt: "lang_lt"
+  lr_lithuanian: "lang_lt"
+  # Swedish
+  lr_sv: "lang_sv"
+  lr_swedish: "lang_sv"
+  # Spanish
+  lr_es: "lang_es"
+  lr_spanish: "lang_es"
+  # Latvian
+  lr_lv: "lang_lv"
+  lr_latvian: "lang_lv"
+  # Turkish
+  lr_tr: "lang_tr"
+  lr_turkish: "lang_tr"
+  # Estonian
+  lr_et: "lang_et"
+  lr_estonian: "lang_et"
+  # Dutch
+  lr_nl: "lang_nl"
+  lr_dutch: "lang_nl"
+  # Chinese (Simplified)
+  lr_zh_cn: "lang_zh-CN"
+  lr_chinese_simplified: "lang_zh-CN"
+  # Finnish
+  lr_fi: "lang_fi"
+  lr_finnish: "lang_fi"
+  # Norwegian
+  lr_no: "lang_no"
+  lr_norwegian: "lang_no"
+  # Chinese (Traditional)
+  lr_zh_tw: "lang_zh-TW"
+  lr_chinese_traditional: "lang_zh-TW"
+  # French
+  lr_fr: "lang_fr"
+  lr_french: "lang_fr"
+  # Croatian
+  lr_hr: "lang_hr"
+  lr_croatian: "lang_hr"
+all: *common_data

data/lib/scrapedo/config/google_builder/time_period.yml

@@ -0,0 +1,7 @@
+common: &common_data
+  hour: "last_hour"
+  today: "last_day"
+  week: "last_week"
+  month: "last_month"
+  year: "last_year"
+all: *common_data