hydra-crawler 1.4.6 → 2.0.2

package/config/complex-english.json

@@ -1,400 +0,0 @@
-{
-	"an absence of": [ "no", "none" ],
-	"abundance": [ "enough", "plenty", "a lot", "many" ],
-	"accede to": [ "allow", "agree to" ],
-	"accelerate": [ "speed up" ],
-	"accentuate": [ "stress" ],
-	"accommodation": [ "where you live", "home" ],
-	"accompanying": [ "with" ],
-	"accomplish": [ "do", "finish" ],
-	"according to our records": [ "our records show" ],
-	"accordingly": [ "in line with this", "so" ],
-	"acknowledge": [ "thank you for" ],
-	"acquaint yourself with": [ "find out about", "read" ],
-	"acquiesce": [ "agree" ],
-	"acquire": [ "buy", "get" ],
-	"additional": [ "extra", "more" ],
-	"adjacent": [ "next to" ],
-	"adjustment": [ "change", "alteration" ],
-	"admissible": [ "allowed", "acceptable" ],
-	"advantageous": [ "useful", "helpful" ],
-	"advise": [ "tell", "say" ],
-	"affix": [ "add", "write", "fasten", "stick on", "fix to" ],
-	"afford an opportunity": [ "let", "allow" ],
-	"afforded": [ "given" ],
-	"aforesaid": [ "this", "earlier in this document" ],
-	"aggregate": [ "total" ],
-	"aligned": [ "lined up", "in line" ],
-	"alleviate": [ "ease", "reduce" ],
-	"allocate": [ "divide", "share", "give" ],
-	"along the lines of": [ "like", "as in" ],
-	"alternative": [ "other" ],
-	"alternatively": [ "or", "on the other hand" ],
-	"ameliorate": [ "improve", "help" ],
-	"amendment": [ "change" ],
-	"anticipate": [ "expect" ],
-	"apparent": [ "clear", "plain", "obvious", "seeming" ],
-	"the applicant": [ "you" ],
-	"application": [ "use" ],
-	"appreciable": [ "large", "great" ],
-	"apprise": [ "inform", "tell" ],
-	"appropriate": [ "proper", "right", "suitable" ],
-	"appropriate to": [ "suitable for" ],
-	"approximately": [ "about", "roughly" ],
-	"as a consequence of": [ "because" ],
-	"as of the date of": [ "from" ],
-	"as regards": [ "about", "on the subject of" ],
-	"ascertain": [ "find out" ],
-	"assemble": [ "build", "gather", "put together" ],
-	"assistance": [ "help" ],
-	"at an early date": [ "soon" ],
-	"at its discretion": [ "can", "may" ],
-	"at the moment": [ "now" ],
-	"at the present time": [ "now" ],
-	"attempt": [ "try" ],
-	"attend": [ "come to", "go to", "be at" ],
-	"attributable to": [ "due to", "because of" ],
-	"authorise": [ "allow", "let" ],
-	"authority": [ "right", "power", "may" ],
-	"axiomatic": [ "obvious", "goes without saying" ],
-	"belated": [ "late" ],
-	"beneficial": [ "helpful", "useful" ],
-	"bestow": [ "give", "award" ],
-	"breach": [ "break" ],
-	"by means of": [ "by" ],
-	"calculate": [ "work out", "decide" ],
-	"cease": [ "finish", "stop", "end" ],
-	"circumvent": [ "get round", "avoid", "skirt", "circle" ],
-	"clarification": [ "explanation", "help" ],
-	"combine": [ "mix" ],
-	"combined": [ "together" ],
-	"commence": [ "start", "begin" ],
-	"communicate": [ "talk", "write", "telephone", "email" ],
-	"competent": [ "able", "can" ],
-	"compile": [ "make", "collect" ],
-	"complete": [ "fill in", "finish" ],
-	"completion": [ "end" ],
-	"comply with": [ "keep to", "meet" ],
-	"component": [ "part" ],
-	"comprises": [ "is made up of", "includes" ],
-	"it is compulsory": [ "you must" ],
-	"conceal": [ "hide" ],
-	"concerning": [ "about", "on" ],
-	"conclusion": [ "end" ],
-	"concur": [ "agree" ],
-	"condition": [ "rule" ],
-	"consequently": [ "so" ],
-	"considerable": [ "great", "important" ],
-	"constitutes": [ "makes up", "forms", "is" ],
-	"construe": [ "interpret" ],
-	"consult": [ "talk to", "meet", "ask" ],
-	"consumption": [ "amount used" ],
-	"contemplate": [ "think about" ],
-	"contrary to": [ "against", "despite" ],
-	"correct": [ "put right" ],
-	"correspond": [ "write" ],
-	"costs the sum of": [ "costs" ],
-	"counter": [ "against" ],
-	"courteous": [ "polite" ],
-	"cumulative": [ "added up", "added together" ],
-	"currently": [ "now" ],
-	"customary": [ "usual", "normal" ],
-	"deduct": [ "take off", "take away" ],
-	"deem to be": [ "treat as" ],
-	"defer": [ "put off", "delay" ],
-	"deficiency": [ "lack of" ],
-	"delete": [ "cross out" ],
-	"demonstrate": [ "show", "prove" ],
-	"denote": [ "show" ],
-	"depict": [ "show" ],
-	"designate": [ "point out", "show", "name" ],
-	"desire": [ "wish", "want" ],
-	"despatch or dispatch": [ "send", "post" ],
-	"despite the fact that": [ "though", "although" ],
-	"determine": [ "decide", "work out", "set", "end" ],
-	"detrimental": [ "harmful", "damaging" ],
-	"difficulties": [ "problems" ],
-	"diminish": [ "lessen", "reduce" ],
-	"disburse": [ "pay", "pay out" ],
-	"discharge": [ "carry out" ],
-	"disclose": [ "tell", "show" ],
-	"disconnect": [ "cut off", "unplug" ],
-	"discontinue": [ "stop", "end" ],
-	"discrete": [ "separate" ],
-	"discuss": [ "talk about" ],
-	"disseminate": [ "spread" ],
-	"documentation": [ "papers", "documents" ],
-	"domiciled in": [ "living in" ],
-	"dominant": [ "main" ],
-	"due to the fact that": [ "because", "as" ],
-	"duration": [ "time", "life" ],
-	"during which time": [ "while" ],
-	"dwelling": [ "home" ],
-	"economical": [ "cheap", "good value" ],
-	"eligible": [ "allowed", "qualified" ],
-	"elucidate": [ "explain", "make clear" ],
-	"emphasise": [ "stress" ],
-	"empower": [ "allow", "let" ],
-	"enable": [ "allow" ],
-	"enclosed": [ "inside", "with" ],
-	"please find enclosed": [ "I enclose" ],
-	"encounter": [ "meet" ],
-	"endeavour": [ "try" ],
-	"enquire": [ "ask" ],
-	"enquiry": [ "question" ],
-	"ensure": [ "make sure" ],
-	"entitlement": [ "right" ],
-	"envisage": [ "expect", "imagine" ],
-	"equivalent": [ "equal", "the same" ],
-	"erroneous": [ "wrong" ],
-	"establish": [ "show", "find out", "set up" ],
-	"evaluate": [ "test", "check" ],
-	"evince": [ "show", "prove" ],
-	"ex officio": [ "because of his or her position" ],
-	"exceptionally": [ "only when", "in this case" ],
-	"excessive": [ "too many", "too much" ],
-	"exclude": [ "leave out" ],
-	"excluding": [ "apart from", "except" ],
-	"exclusively": [ "only" ],
-	"exempt from": [ "free from" ],
-	"expedite": [ "hurry", "speed up" ],
-	"expeditiously": [ "as soon as possible", "quickly" ],
-	"expenditure": [ "spending" ],
-	"expire": [ "run out" ],
-	"extant": [ "current", "in force" ],
-	"extremity": [ "limit" ],
-	"fabricate": [ "make", "make up" ],
-	"facilitate": [ "help", "make possible" ],
-	"factor": [ "reason" ],
-	"failure to": [ "if you do not" ],
-	"finalise": [ "end", "finish" ],
-	"following": [ "after" ],
-	"for the duration of": [ "during", "while" ],
-	"for the purpose of": [ "to", "for" ],
-	"for the reason that": [ "because" ],
-	"formulate": [ "plan", "devise" ],
-	"forthwith": [ "now", "at once" ],
-	"forward": [ "send" ],
-	"frequently": [ "often" ],
-	"furnish": [ "give" ],
-	"further to": [ "after", "following" ],
-	"furthermore": [ "then", "also", "and" ],
-	"generate": [ "produce", "give", "make" ],
-	"give consideration to": [ "consider", "think about" ],
-	"grant": [ "give" ],
-	"henceforth": [ "from now on", "from today" ],
-	"hereby": [ "now", "by this" ],
-	"herein": [ "here" ],
-	"hereinafter": [ "after this" ],
-	"hereof": [ "of this" ],
-	"hereto": [ "to this" ],
-	"heretofore": [ "until now", "previously" ],
-	"hereunder": [ "below" ],
-	"herewith": [ "with this" ],
-	"hitherto": [ "until now" ],
-	"hold in abeyance": [ "wait", "postpone" ],
-	"hope and trust": [ "hope", "trust" ],
-	"if and when": [ "if", "when" ],
-	"illustrate": [ "show", "explain" ],
-	"immediately": [ "at once", "now" ],
-	"implement": [ "carry out", "do" ],
-	"imply": [ "suggest", "hint at" ],
-	"in a number of cases": [ "some cases" ],
-	"in accordance with": [ "as under", "in line with", "because of" ],
-	"in addition to": [ "and", "as well as", "also" ],
-	"in advance": [ "before" ],
-	"in case of": [ "if" ],
-	"in conjunction with": [ "and", "with" ],
-	"in connection with": [ "for", "about" ],
-	"in consequence": [ "because", "as a result" ],
-	"in excess of": [ "more than" ],
-	"in lieu of": [ "instead of" ],
-	"in order that": [ "so that" ],
-	"in receipt of": [ "get", "have", "receive" ],
-	"in relation to": [ "about" ],
-	"in respect of": [ "about", "for" ],
-	"in the absence of": [ "without" ],
-	"in the course of": [ "while", "during" ],
-	"in the event of/that": [ "if" ],
-	"in the majority of instances": [ "most", "mostly" ],
-	"in the near future": [ "soon" ],
-	"in the neighbourhood of": [ "about", "around" ],
-	"in view of the fact that": [ "as", "because" ],
-	"inappropriate": [ "wrong", "unsuitable" ],
-	"inception": [ "start", "beginning" ],
-	"incorporating": [ "which includes" ],
-	"incur": [ "have to pay", "owe" ],
-	"indicate": [ "show", "suggest" ],
-	"inform": [ "tell" ],
-	"initially": [ "at first" ],
-	"initiate": [ "begin", "start" ],
-	"insert": [ "put in" ],
-	"instances": [ "cases" ],
-	"intend to": [ "will" ],
-	"intimate": [ "say", "hint" ],
-	"irrespective of": [ "despite", "even if" ],
-	"is of the opinion": [ "thinks" ],
-	"issue": [ "give", "send" ],
-	"it is known that": [ "I/we know that" ],
-	"jeopardise": [ "risk", "threaten" ],
-	"a large number of": [ "many", "most" ],
-	"locality": [ "place", "area" ],
-	"locate": [ "find", "put" ],
-	"magnitude": [ "size" ],
-	"it is mandatory": [ "you must" ],
-	"manner": [ "way" ],
-	"manufacture": [ "make" ],
-	"marginal": [ "small", "slight" ],
-	"material": [ "relevant" ],
-	"materialise": [ "happen", "occur" ],
-	"may in the future": [ "may", "might", "could" ],
-	"merchandise": [ "goods" ],
-	"mislay": [ "lose" ],
-	"modification": [ "change" ],
-	"moreover": [ "and", "also", "as well" ],
-	"negligible": [ "very small" ],
-	"nevertheless": [ "but", "however", "even so" ],
-	"notify": [ "tell", "let us know", "let you know" ],
-	"notwithstanding": [ "even if", "despite", "still", "yet" ],
-	"numerous": [ "many" ],
-	"objective": [ "aim", "goal" ],
-	"it is obligatory": [ "you must" ],
-	"obtain": [ "get", "receive" ],
-	"occasioned by": [ "caused by", "because of" ],
-	"on behalf of": [ "for" ],
-	"on numerous occasions": [ "often" ],
-	"on request": [ "if you ask" ],
-	"on the grounds that": [ "because" ],
-	"on the occasion that": [ "when", "if" ],
-	"operate": [ "work", "run" ],
-	"optimum": [ "best", "ideal" ],
-	"option": [ "choice" ],
-	"ordinarily": [ "normally", "usually" ],
-	"otherwise": [ "or" ],
-	"outstanding": [ "unpaid" ],
-	"owing to": [ "because of" ],
-	"partially": [ "partly" ],
-	"participate": [ "join in", "take part" ],
-	"particulars": [ "details", "facts" ],
-	"per annum": [ "a year" ],
-	"perform": [ "do" ],
-	"permissible": [ "allowed" ],
-	"permit": [ "let", "allow" ],
-	"personnel": [ "people", "staff" ],
-	"persons": [ "people", "anyone" ],
-	"peruse": [ "read", "read carefully", "look at" ],
-	"place": [ "put" ],
-	"possess": [ "have", "own" ],
-	"possessions": [ "belongings" ],
-	"practically": [ "almost", "nearly" ],
-	"predominant": [ "main" ],
-	"prescribe": [ "set", "fix" ],
-	"preserve": [ "keep", "protect" ],
-	"previous": [ "earlier", "before", "last" ],
-	"principal": [ "main" ],
-	"prior to": [ "before" ],
-	"proceed": [ "go ahead" ],
-	"procure": [ "get", "obtain", "arrange" ],
-	"profusion of": [ "plenty", "too many" ],
-	"prohibit": [ "ban", "stop" ],
-	"projected": [ "estimated" ],
-	"prolonged": [ "long" ],
-	"promptly": [ "quickly", "at once" ],
-	"promulgate": [ "advertise", "announce" ],
-	"proportion": [ "part" ],
-	"provide": [ "give" ],
-	"provided that": [ "if", "as long as" ],
-	"provisions": [ "rules", "terms" ],
-	"proximity": [ "closeness", "nearness" ],
-	"purchase": [ "buy" ],
-	"pursuant to": [ "under", "because of", "in line with" ],
-	"reconsider": [ "think again about", "look again at" ],
-	"reduce": [ "cut" ],
-	"reduction": [ "cut" ],
-	"referred to as": [ "called" ],
-	"refers to": [ "talks about", "mentions" ],
-	"have regard to": [ "take into account" ],
-	"regarding": [ "about", "on" ],
-	"regulation": [ "rule" ],
-	"reimburse": [ "repay", "pay back" ],
-	"reiterate": [ "repeat", "restate" ],
-	"relating to": [ "about" ],
-	"remain": [ "stay" ],
-	"remainder": [ "the rest", "what is left" ],
-	"remittance": [ "payment" ],
-	"remuneration": [ "pay", "wages", "salary" ],
-	"render": [ "make", "give", "send" ],
-	"report": [ "tell" ],
-	"represents": [ "shows", "stands for", "is" ],
-	"request": [ "ask", "question" ],
-	"require": [ "need", "want", "force" ],
-	"requirements": [ "needs", "rules" ],
-	"reside": [ "live" ],
-	"residence": [ "home", "where you live" ],
-	"restriction": [ "limit" ],
-	"retain": [ "keep" ],
-	"review": [ "look at again" ],
-	"revised": [ "new", "changed" ],
-	"said/such/same": [ "the", "this", "that" ],
-	"scrutinise": [ "read carefully" ],
-	"select": [ "choose" ],
-	"settle": [ "pay" ],
-	"similarly": [ "also", "in the same way" ],
-	"solely": [ "only" ],
-	"specified": [ "given", "written", "set" ],
-	"state": [ "say", "tell us", "write down" ],
-	"statutory": [ "legal", "by law" ],
-	"subject to": [ "depending on", "under", "keeping to" ],
-	"submit": [ "send", "give" ],
-	"subsequent to/upon": [ "after" ],
-	"subsequently": [ "later" ],
-	"substantial": [ "large", "great", "a lot of" ],
-	"substantially": [ "more or less" ],
-	"sufficient": [ "enough" ],
-	"supplement": [ "go with", "add to" ],
-	"supplementary": [ "extra", "more" ],
-	"supply": [ "give", "sell", "deliver" ],
-	"the tenant": [ "you" ],
-	"terminate": [ "stop", "end" ],
-	"that being the case": [ "if so" ],
-	"the question as to whether": [ "whether" ],
-	"thereafter": [ "then", "afterwards" ],
-	"thereby": [ "by that", "because of that" ],
-	"therein": [ "in that", "there" ],
-	"thereof": [ "of that" ],
-	"thereto": [ "to that" ],
-	"thus": [ "so", "therefore" ],
-	"to date": [ "so far", "up to now" ],
-	"to the extent that": [ "if", "when" ],
-	"transfer": [ "change", "move" ],
-	"transmit": [ "send" ],
-	"ultimately": [ "in the end", "finally" ],
-	"unavailability": [ "lack of" ],
-	"undernoted": [ "the following" ],
-	"undersigned": [ "I", "we" ],
-	"undertake": [ "agree", "promise", "do" ],
-	"uniform": [ "same", "similar" ],
-	"unilateral": [ "one-sided", "one-way" ],
-	"unoccupied": [ "empty" ],
-	"until such time": [ "until" ],
-	"utilisation": [ "use" ],
-	"utilise": [ "use" ],
-	"variation": [ "change" ],
-	"virtually": [ "almost" ],
-	"visualise": [ "see", "predict" ],
-	"ways and means": [ "ways" ],
-	"we have pleasure in": [ "we are glad to" ],
-	"whatsoever": [ "whatever", "what", "any" ],
-	"whensoever": [ "when" ],
-	"whereas": [ "but" ],
-	"whether or not": [ "whether" ],
-	"with a view to": [ "to", "so that" ],
-	"with effect from": [ "from" ],
-	"with reference to": [ "about" ],
-	"with regard to": [ "about", "for" ],
-	"with respect to": [ "about", "for" ],
-	"with the minimum of delay": [ "quickly" ],
-	"you are requested": [ "please" ],
-	"your attention is drawn to": [ "please see", "please note" ],
-	"zone": [ "area", "region" ]
-}

package/config/hydra-auth.json

@@ -1,8 +0,0 @@
-{
-	"database": {
-		"user": "",
-		"password": "",
-		"name": "hydra",
-		"host": "localhost"
-	}
-}

package/config/hydra-crawler.json

@@ -1,84 +0,0 @@
-{
-	"express": {
-		"port": 8082,
-		"path": "/"
-	},
-	"angular": {},
-	"hydra": {
-		"startUrl": "https://www.manchester.ac.uk/"
-	},
-	"crawl": {
-		"findNewAvailableDelay": 20000,
-		"maxCrawlers": 50,
-		"maxFetchesPerCrawl": 250,
-		"availableStrategyThreshold": 100,
-		"betweenFetchDelay": 15000,
-		"randomisation": 0.05,
-		"connectTimeout": 30000,
-		"maxFileSize": 4194304,
-		"maxFailedTtl": 1,
-		"keepHeaders": [
-			"content-type:string", "last-modified:date", "date:date", "server:string", 
-			"x-powered-by:string", "content-length:number", "location:string"
-		],
-		"treatTimeoutDnsAsDead": true
-	},
-	"robots": {
-		"ignoreWholeSiteDirectives": true
-	},
-	"parsers": {
-		"badWords": {
-			"enabled": true,
-			"caseSensitive": false,
-			"allowHyphenatedWords": true,
-			"dictionary": "bad-words.json",
-			"maxDataBytes": 512000
-		},
-		"spelling": {
-			"enabled": true,
-			"caseSensitive": false,
-			"allowHyphenatedWords": true,
-			"dictionary": "spelling.json",
-			"maxDataBytes": 512000
-		},
-		"complexEnglish": {
-			"enabled": true,
-			"caseSensitive": false,
-			"dictionary": "complex-english.json",
-			"maxDataBytes": 512000
-		},
-		"accessibilityMetrics": {
-			"enabled": true,
-			"paragraphWordsThreshold": 10,
-			"maxDataBytes": 512000
-		},
-		"hyperlinks": {
-			"enabled": true,
-			"sources": [
-				"img[src]", "a[href]", "link[href]", "script[src]", 
-				"form[action]", "iframe[src]", "embed[src]", "source[src]", 
-				"audio[src]", "track[src]", "video[src]", "object[src]",
-				"frame[src]"
-			],
-			"maxDataBytes": 1024000
-		},
-		"phpError": {
-			"enabled": true,
-			"maxDataBytes": 1024000
-		},
-		"aspError": {
-			"enabled": true,
-			"maxDataBytes": 1024000
-		},
-		"imageTags": {
-			"enabled": true,
-			"maxDataBytes": 1024000
-		},
-		"jpeg": {
-			"enabled": true
-		},
-		"server": {
-			"enabled": true
-		}
-	}
-}

package/config/list-allow.json

@@ -1,171 +0,0 @@
-[
-	{ "hostname": "(^|\\.)agocg\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)alma\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)angles\\.eu\\.org$" },
-	{ "hostname": "(^|\\.)ashtoninstitute\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)badbir\\.org$" },
-	{ "hostname": "(^|\\.)bcrdstudy\\.org$" },
-	{ "hostname": "(^|\\.)bilagbr\\.org$" },
-	{ "hostname": "(^|\\.)biospec\\.net$" },
-	{ "hostname": "(^|\\.)biotrains\\.eu$" },
-	{ "hostname": "(^|\\.)boystomenproject\\.com$" },
-	{ "hostname": "(^|\\.)braggss\\.co\\.uk$" },
-	{ "hostname": "(^|\\.)brazil4africa\\.org$" },
-	{ "hostname": "(^|\\.)brin\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)bsrbr\\.org$" },
-	{ "hostname": "(^|\\.)cam-man-cic\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)capturingthegains\\.org$" },
-	{ "hostname": "(^|\\.)ccalc\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)ccsr\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)chancellorscollection\\.co\\.uk$" },
-	{ "hostname": "(^|\\.)chancellorshotel\\.co\\.uk$" },
-	{ "hostname": "(^|\\.)chethams\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)chronicpoverty\\.org$" },
-	{ "hostname": "(^|\\.)citizenscientist\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)climatejust\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)clinicalcodes\\.org$" },
-	{ "hostname": "(^|\\.)coebio3\\.org$" },
-	{ "hostname": "(^|\\.)confercare-online\\.co\\.uk$" },
-	{ "hostname": "(^|\\.)consultationskillsforpharmacy\\.com$" },
-	{ "hostname": "(^|\\.)cpi\\.umist\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)cppe\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)cresc\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)cruklungcentre\\.org$" },
-	{ "hostname": "(^|\\.)csar\\.cfs\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)data\\.synbiochem\\.co\\.uk$" },
-	{ "hostname": "(^|\\.)digital60\\.org$" },
-	{ "hostname": "(^|\\.)domesticabuserisk\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)dtors\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)eastbordnet\\.org$" },
-	{ "hostname": "(^|\\.)effective-states\\.org$" },
-	{ "hostname": "(^|\\.)egov4dev\\.org$" },
-	{ "hostname": "(^|\\.)engager-energy\\.net$" },
-	{ "hostname": "(^|\\.)epspoetry\\.com$" },
-	{ "hostname": "(^|\\.)epta\\.eu\\.org$" },
-	{ "hostname": "(^|\\.)esciencelab\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)esilconference2018\\.com$" },
-	{ "hostname": "(^|\\.)esrcmanchesterfest\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)ethnicity\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)everydayparticipation\\.org$" },
-	{ "hostname": "(^|\\.)farseeingresearch\\.eu$" },
-	{ "hostname": "(^|\\.)festivalofpublichealth\\.co\\.uk$" },
-	{ "hostname": "(^|\\.)foodallergens\\.info$" },
-	{ "hostname": "(^|\\.)futuredams\\.org$" },
-	{ "hostname": "(^|\\.)gamingsociety\\.co\\.uk$" },
-	{ "hostname": "(^|\\.)graduatetalentpool\\.direct\\.gov\\.uk$" },
-	{ "hostname": "(^|\\.)graduatetalentpoolsearch\\.direct\\.gov\\.uk$" },
-	{ "hostname": "(^|\\.)hcri\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)healtex\\.org$" },
-	{ "hostname": "(^|\\.)hecsu\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)hedd\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)hewitt\\.ddns\\.net$" },
-	{ "hostname": "(^|\\.)homeoffshore\\.org$" },
-	{ "hostname": "(^|\\.)hope-academic\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)humanitarian-quest\\.org$" },
-	{ "hostname": "(^|\\.)humanitarianhistory\\.org$" },
-	{ "hostname": "(^|\\.)iapetus\\.phy\\.umist\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)icam-online\\.org$" },
-	{ "hostname": "(^|\\.)icuh2014\\.com$" },
-	{ "hostname": "(^|\\.)iid2\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)ippg\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)ipta4gw\\.org$" },
-	{ "hostname": "(^|\\.)ist2018\\.org$" },
-	{ "hostname": "(^|\\.)jodcast\\.net$" },
-	{ "hostname": "(^|\\.)jodrellbank\\.net$" },
-	{ "hostname": "(^|\\.)leap\\.eu\\.org$" },
-	{ "hostname": "(^|\\.)ltea\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)lupusmasterplans\\.org$" },
-	{ "hostname": "(^|\\.)ma\\.umist\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)maas\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)mabuls\\.net$" },
-	{ "hostname": "(^|\\.)mahsc\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)mahsc\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)man\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)manbio\\.uk$" },
-	{ "hostname": "(^|\\.)manchester\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)manchester\\.nightline\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)manchestercityofliterature\\.com$" },
-	{ "hostname": "(^|\\.)manchesterjmce\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)mancmn\\.org$" },
-	{ "hostname": "(^|\\.)manjasuta\\.co\\.uk$" },
-	{ "hostname": "(^|\\.)manunicareersblog\\.com$" },
-	{ "hostname": "(^|\\.)mbclick\\.com$" },
-	{ "hostname": "(^|\\.)mbs\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)mcc\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)mcisb\\.org$" },
-	{ "hostname": "(^|\\.)meeting\\.co\\.uk$" },
-	{ "hostname": "(^|\\.)merc\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)merlin\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)methodbox\\.org$" },
-	{ "hostname": "(^|\\.)methodsnorthwest\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)mettel-prover\\.org$" },
-	{ "hostname": "(^|\\.)mib\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)mirbase\\.org$" },
-	{ "hostname": "(^|\\.)mobile4\\.phy\\.umist\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)multi-story-shipley\\.co\\.uk$" },
-	{ "hostname": "(^|\\.)mygrid\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)n8hpc\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)nactem\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)narg\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)nationalplanning\\.org$" },
-	{ "hostname": "(^|\\.)ncess\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)ndtms\\.net$" },
-	{ "hostname": "(^|\\.)neighbourhoodsanddementia\\.org$" },
-	{ "hostname": "(^|\\.)nhssalfordrd\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)niccd\\.org$" },
-	{ "hostname": "(^|\\.)npcrdc\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)ntec\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)nwcdtp\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)nwdtc\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)ohrn\\.nhs\\.uk$" },
-	{ "hostname": "(^|\\.)omic\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)parkinsonspain\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)performancefootprint\\.co\\.uk$" },
-	{ "hostname": "(^|\\.)pit-sig\\.uk$" },
-	{ "hostname": "(^|\\.)politicalperspectives\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)prepska\\.org$" },
-	{ "hostname": "(^|\\.)preventra\\.net$" },
-	{ "hostname": "(^|\\.)projectcode\\.net$" },
-	{ "hostname": "(^|\\.)prospects\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)psort\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)rachelgibson\\.info$" },
-	{ "hostname": "(^|\\.)rams-study\\.net$" },
-	{ "hostname": "(^|\\.)researchdirectorate\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)researchobject\\.org$" },
-	{ "hostname": "(^|\\.)risingpowers\\.net$" },
-	{ "hostname": "(^|\\.)romanistudies\\.eu$" },
-	{ "hostname": "(^|\\.)royce\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)safi-research\\.org$" },
-	{ "hostname": "(^|\\.)salfordresearch\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)scalpl\\.org$" },
-	{ "hostname": "(^|\\.)sciencex\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)servicecatalographer\\.org$" },
-	{ "hostname": "(^|\\.)servicecatalogue\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)springsustainability\\.org$" },
-	{ "hostname": "(^|\\.)ssc-esos\\.net$" },
-	{ "hostname": "(^|\\.)structuralconcepts\\.org$" },
-	{ "hostname": "(^|\\.)structuralfiresafety\\.com$" },
-	{ "hostname": "(^|\\.)structuralfiresafety\\.org$" },
-	{ "hostname": "(^|\\.)supergen-bioenergy\\.net$" },
-	{ "hostname": "(^|\\.)surrealismcentre\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)sustainable-systems\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)swebnet\\.co\\.umist\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)tableyhouse\\.co\\.uk$" },
-	{ "hostname": "(^|\\.)tarn\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)tastemcr\\.com$" },
-	{ "hostname": "(^|\\.)taverna\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)thelearningpharmacy\\.com$" },
-	{ "hostname": "(^|\\.)themanchesterreview\\.co\\.uk$" },
-	{ "hostname": "(^|\\.)tina-vision\\.net$" },
-	{ "hostname": "(^|\\.)uhsmacademylibrary\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)umist\\.ac\\.uk$" },
-	{ "hostname": "(^|\\.)umss\\.co\\.uk$" },
-	{ "hostname": "(^|\\.)umucu\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)urbantippingpoint\\.org$" },
-	{ "hostname": "(^|\\.)urhis\\.eu$" },
-	{ "hostname": "(^|\\.)utopiadocs\\.com$" },
-	{ "hostname": "(^|\\.)view\\.commonwl\\.org$" },
-	{ "hostname": "(^|\\.)viewit\\.org\\.uk$" },
-	{ "hostname": "(^|\\.)watchingdance\\.org$" },
-	{ "hostname": "(^|\\.)when2go-study\\.co\\.uk$" }
-]