@jayesol/jayeson.lib.delivery 2.0.6 → 2.0.7

package/lib/Subscriber.js

@@ -1 +1,1083 @@
-var coreMsges=require("./CoreMessages"),deliveryStack=require("./DeliveryStack"),_=require("underscore"),EventEmitter=require("eventemitter3"),streamFinder=require("@jayesol/jayeson.lib.streamfinder"),Discoverer=streamFinder.Discoverer,util=require("./util"),_c=util._c,DefaultClient=deliveryStack.DefaultClient,EPEvent=deliveryStack.EPEvent,EPConnectedEvent=deliveryStack.EPConnectedEvent,EPConnectionError=deliveryStack.EPConnectionError,EPDisconnectedEvent=deliveryStack.EPDisconnectedEvent,IEndPointEventSource=deliveryStack.IEndPointEventSource,generateUUID=deliveryStack.generateUUID,Util=deliveryStack.Util,MessageWrapper=deliveryStack.MessageWrapper,StreamRegistryMessageGroup=coreMsges.StreamRegistryMessageGroup,EmptyMessageClass=coreMsges.EmptyMessageClass,StringMessageClass=coreMsges.StringMessageClass,JSonMessageClass=coreMsges.JSonMessageClass,StreamRegistryRequest=coreMsges.StreamRegistryRequest,StreamRegistryResponse=coreMsges.StreamRegistryResponse,AuthGroup=coreMsges.AuthGroup,GenericMessageGroup=coreMsges.GenericMessageGroup,KeepAliveMessageGroup=coreMsges.KeepAliveMessageGroup,TicketRenew=coreMsges.TicketRenew;function SubscriberConfig(){this.autoDiscover=!0,this.streamFinder="",this.username="",this.password="",this.discoveryPattern={},this.processors={},this.schemes=["ws"],this.ticketRenewalInterval=84e4,this.reconnectionInterval=3e3,this.exclusionThreshold=3,this.feedScope="",this.discoverByScope=!0}var DATA_NODE_COUNTER=0;function DataNode(e,t,s,o,i,n){this.client=null,this.streams={},this.id=e,this.counter=DATA_NODE_COUNTER++,console.log("[DataNode] Node ",e," counter ",this.counter," is created!"),this.state=DataNode.State.CREATED,this.connection=o.connection,this.toBeConsumed=[],this.consumeStream(o),this.clientId=t,this.ticket=s,this.consumeTaskTimeoutID=0,this.eventBus=i,this.sConf=n,this.removalPromises={},this.consumptionPromises={}}function Subscriber(e){IEndPointEventSource.call(this),this.conf=new SubscriberConfig,_.extend(this.conf,e),this.consuming={},this.nodes={},this.currentAvailableSources=null,this.hasUsedCurrentSources=!1,this.clientId=this.uuid(),this.ticket=null,this.eventBus=new EventEmitter,this.eventBus.on(DataNode.Event.RECONNECTION_ERROR,_.bind(this.onNodeReconnectionError,this)),this.eventBus.on(DataNode.Event.STATE,_.bind(this.onNodeStateChanged,this)),this.eventBus.on(DataNode.Event.CONSUMPTION_START,_.bind(this.onStreamConsumption,this)),this.eventBus.on(DataNode.Event.CONSUMPTION_ERROR,_.bind(this.onStreamError,this)),this.eventBus.on(DataNode.Event.CONSUMPTION_STOP,_.bind(this.onStreamStop,this))}function StreamConsumptionWrapper(e,t){this.state=e,this.node=t}function ConsumptionStart(e,t){EPEvent.call(this,e._ep),this.client=e,this.streams=t}function ConsumptionStop(e,t){EPEvent.call(this,e._ep),this.client=e,this.streams=t}function ConsumptionError(e,t){EPEvent.call(this,e._ep),this.client=e,this.streams=t}DataNode.State={CREATED:1,CONNECTING:2,CONNECTED:3,RECONNECTING:4,DISCONNECTED:5,DESTROYED:6},DataNode.Event={STATE:"DN_STATE",RECONNECTION_ERROR:"DN_RE_ERROR",CONSUMPTION_ERROR:"DN_CONSUMPTION_ERROR",CONSUMPTION_START:"DN_CONSUMPTION_START",CONSUMPTION_STOP:"DN_CONSUMPTION_STOP"},_c(DataNode,{removeStream:function(n,r,a){var c=Util._streamIdFromNames(n,r),e=_.bind(function(e){var t,s=_.bind(function(e){if(e&&this.streams[n]){var t;for(t=0;t<this.streams[n].length&&r!=this.streams[n][t].stream;t++);t<this.streams[n].length&&this.streams[n].splice(t,1),0==this.streams[n].length&&delete this.streams[n],_.isEmpty(this.streams)&&this.destroy()}return e},this);if(this.state==DataNode.State.CONNECTED&&a){t=Util._newPromise(),this.removalPromises[c]=t;var o=new StreamRegistryRequest(n,[r],StreamRegistryRequest.DEREGISTRATION);this.client.send(new MessageWrapper(o,StreamRegistryMessageGroup.REQUEST));var i=this;return window.setTimeout(function(){if(void 0!==i.removalPromises[c]){console.warn("Timeout waiting for deregistration response, proceeding to treat stream "+n+":"+r+" as deregistered for "+i.connection.hostname+":"+i.connection.port);var e={};e[r]=StreamRegistryResponse.DEREGISTRATION_TIMEOUT;var t=new StreamRegistryResponse(n,e,StreamRegistryResponse.DEREGISTRATION);i._processDeregistrationResult(t)}},7e3),t.then(s)}return t=Promise.resolve(s(!0))},this);return this.consumptionPromises&&void 0!==this.consumptionPromises[c]?this.consumptionPromises[c].then(e):e(!0)},hasStream:function(e,t){return!!this.streams[e]&&void 0!==(t=_.each(this.streams[e],function(e){return e.stream==t}))},consumeStream:function(e){console.log("Node "+this.id+" is going to consume "+e.group+"_"+e.stream),this.toBeConsumed.push(e),this.streams[e.group]||(this.streams[e.group]=[]),this.streams[e.group].push(e),0==this.consumeTaskTimeoutID&&this.state==DataNode.State.CONNECTED&&(this.consumeTaskTimeoutID=window.setTimeout(_.bind(this.consumeTask,this),100))},_consumeStreamsOnReconnected:function(){for(var e in _.isEmpty(this.streams)&&this.destroy(),this.streams)if(this.streams.hasOwnProperty(e))for(var t=0;t<this.streams[e].length;t++)this.toBeConsumed.push(this.streams[e][t])},connect:function(){console.log("[DataNode] Id ",this.id," counter ",this.counter," is going to connect. Current state ",this.state),this.client=new DefaultClient({url:this._getUrl(this.connection),msgClass:AuthGroup.classById(1),params:{clientId:this.clientId,ticket:this.ticket,feedScope:this.sConf.feedScope}},KeepAliveMessageGroup),this.client.attachListener(this),this.client.enableKeepAlive(),this.client.registerGroup(AuthGroup,this),this.client.registerGroup(StreamRegistryMessageGroup,this),this.client.connect()},reconnect:function(){console.log("[DataNode] Id ",this.id," counter ",this.counter," is going to re-connect. Current state ",this.state),this.client.config.params={clientId:this.clientId,ticket:this.ticket,feedScope:this.sConf.feedScope},this.client.connect()},_getUrl:function(e){return e.protocol+"://"+e.hostname+(void 0!==e.port?":"+e.port:"")+(void 0!==e.path?e.path:"")},consumeTask:function(){var n=this;if(this.state==DataNode.State.CONNECTED){var e=_.groupBy(this.toBeConsumed,function(e){return e.group});this.toBeConsumed=[],this.consumeTaskTimeoutID=0,_.each(e,function(e,t){var s=_.filter(e,function(e){var t=Util._streamId(e);return void 0===n.removalPromises[t]&&void 0===n.consumptionPromises[t]}),o=(_.reduce(s,function(e,t){var s=Util._streamId(t);return e[s]=Util._newPromise(),e[s].then(function(e){console.log("Consumption result of ",s," : successful = ",e)}),e},n.consumptionPromises),_.map(s,function(e){return e.stream})),i=new StreamRegistryRequest(t,o,StreamRegistryRequest.REGISTRATION);n.client.send(new MessageWrapper(i,StreamRegistryMessageGroup.REQUEST))})}},onEvent:function(e){if(console.log("[DataNode] Event from the stack client",e),e instanceof EPConnectedEvent)this.state===DataNode.State.DISCONNECTED&&this._consumeStreamsOnReconnected(),this.setState(DataNode.State.CONNECTED),console.log("[DataNode] Connect / Re-connect to ",this.connection.hostname,":",this.connection.port,"successfully. Consumption will be started!"),this.consumeTaskTimeoutID=window.setTimeout(_.bind(this.consumeTask,this),100);else if(e instanceof EPDisconnectedEvent||e instanceof EPConnectionError){if(this.state==DataNode.State.DESTROYED)return;this.setState(DataNode.State.DISCONNECTED),console.log("Connect / Re-connect to ",this.connection.hostname,":",this.connection.port," with error. Retry again later."),_.each(this.consumptionPromises,function(e,t){e.resolve(!1)}),_.each(this.removalPromises,function(e,t){e.resolve(!0)}),delete this.consumptionPromises,delete this.removalPromises,this.consumptionPromises={},this.removalPromises={},this.state!=DataNode.State.DESTROYED?window.setTimeout(_.bind(this.reconnect,this),this.sConf.reconnectionInterval):console.log("Data node ",this.id," has been destroyed. Stop reconnecting."),0!==this.consumeTaskTimeoutID&&window.clearTimeout(this.consumeTaskTimeoutID),e instanceof EPConnectionError?this.eventBus.emit(DataNode.Event.RECONNECTION_ERROR,this,++this.numFailedReconnection):this.numFailedReconnection=0}},process:function(e){if(e.message instanceof StreamRegistryResponse){var t=e.message;t.responseType==StreamRegistryResponse.REGISTRATION?this._processRegistrationResult(t):t.responseType==StreamRegistryResponse.CONSUMPTION?this._processConsumptionResult(t):t.responseType==StreamRegistryResponse.DEREGISTRATION?this._processDeregistrationResult(t):t.responseType==StreamRegistryResponse.STOP_CONSUMPTION&&this._processStopConsumptionResult(t)}},_resolveConsumptionPromise:function(e,t,s){var o=Util._streamIdFromNames(e,t);void 0!==this.consumptionPromises[o]&&this.consumptionPromises[o].resolve(s),delete this.consumptionPromises[o]},_resolveRemovalPromise:function(e,t,s){var o=Util._streamIdFromNames(e,t);void 0!==this.removalPromises[o]&&(this.removalPromises[o].resolve(s),delete this.removalPromises[o]),delete this.consumptionPromises[o]},_processDeregistrationResult:function(e){var s=this,o=e.messageGroup,i={};_.each(e.streams,function(e,t){e==StreamRegistryResponse.FAILED?s._resolveRemovalPromise(o,t,!1):(s._resolveRemovalPromise(o,t,!0),void 0===i[o]&&(i[o]=[]),i[o].push(t))}),0==_.isEmpty(i)&&(console.log("[DataNode] Emitting consumption error after Deregistration"),this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR,this,i))},_processStopConsumptionResult:function(e){var s=this,o=e.messageGroup,i={};_.each(e.streams,function(e,t){console.log("Stopping stream "+t),s.removeStream(o,t,!1),void 0===i[o]&&(i[o]=[]),i[o].push(t)}),_.isEmpty(i)||this.eventBus.emit(DataNode.Event.CONSUMPTION_STOP,this,i)},_processConsumptionResult:function(e){var s=this,o=e.messageGroup,i={},n={};_.each(e.streams,function(e,t){e==StreamRegistryResponse.FAILED?(s._resolveConsumptionPromise(o,t,!1),s.removeStream(o,t,!0),void 0===n[o]&&(n[o]=[]),n[o].push(t)):(s._resolveConsumptionPromise(o,t,!0),void 0===i[o]&&(i[o]=[]),i[o].push(t))}),0==_.isEmpty(i)&&this.eventBus.emit(DataNode.Event.CONSUMPTION_START,this,i),0==_.isEmpty(n)&&this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR,this,n)},_processRegistrationResult:function(e){var t=this,s=e.messageGroup,o=_.pairs(e.streams).filter(function(e){return e[1]==StreamRegistryResponse.FAILED}).map(_.first);o.forEach(function(e){t._resolveConsumptionPromise(s,e,!1),t.removeStream(s,e,!1)});var i=_.pairs(e.streams).filter(function(e){return e[1]==StreamRegistryResponse.SUCCESS}).map(_.first).filter(function(e){return void 0!==t.sConf.processors[s]||(t._resolveConsumptionPromise(s,e,!1),!1)});if(!_.isEmpty(i)){var n=t.sConf.processors[s];t.client.registerGroup(n.group(),n);var r=new StreamRegistryRequest(s,i,StreamRegistryRequest.CONSUMPTION);t.client.send(new MessageWrapper(r,StreamRegistryMessageGroup.REQUEST))}if(!_.isEmpty(o)){var a={};a[s]=o,console.log("Emitting consumption error due to registration result failed.",a),this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR,this,a)}},setState:function(e){if(e!=this.state){var t=this.state;this.state=e,this.eventBus.emit(DataNode.Event.STATE,this,t)}},renewTicket:function(e){this.ticket=e;var t=new MessageWrapper(new TicketRenew(this.clientId,this.ticket),AuthGroup.TICKET_RENEW);this.state==DataNode.State.CONNECTED&&this.client.send(t)},destroy:function(){this.setState(DataNode.State.DESTROYED),console.log("[DataNode] Disconnect the destroyed data node."),this.client.disconnect()}}),_c(ConsumptionStart,{},EPEvent),_c(ConsumptionStop,{},EPEvent),_c(ConsumptionError,{},EPEvent),Subscriber.Event={ConsumptionStart:ConsumptionStart,ConsumptionStop:ConsumptionStop,ConsumptionError:ConsumptionError},_c(Subscriber,{consumptionState:{PENDING:1,ERROR:3},uuid:function(){return generateUUID()},config:function(){return this.conf},start:function(){var s=new Discoverer(this.config().streamFinder);this.conf.discoverByScope&&(s=new Discoverer(this.config().streamFinder,void 0,this.conf.feedScope)),_.each(this.config().discoveryPattern,function(e,t){_.each(e,function(e){s.discover(t,e)})}),_.each(this.config().schemes,function(e){s.using(e)}),s.on("update",_.bind(this._updateSources,this)),s.on("error",_.bind(this._discovererError,this)),setInterval(_.bind(this._reconnectStreams,this),5e3),this._makeTicket(!0),this.discoverer=s},_makeTicket:function(s){SessionFactory.makeTicket(this.config().streamFinder,this.clientId,_.bind(function(e,t){0!=e.length?(this.ticket=e,console.log("New ticket obtained "+this.ticket),s?this.discoverer.start():_.each(this.nodes,_.bind(function(e,t){window.setTimeout(_.bind(e.renewTicket,e,this.ticket),0)},this)),window.setTimeout(_.bind(this._makeTicket,this),this.config().ticketRenewalInterval)):0!=t.length&&window.setTimeout(_.bind(this._makeTicket,this),1e3)},this))},_updateSources:function(e){console.log("Updating Sources!"),console.log(e),this.oldSources=this.currentAvailableSources,this.currentAvailableSources=e.streamConnections,this.hasUsedCurrentSources=!1;var s={},o={};if(this.oldSources&&0<this.oldSources.length){var i=[],n=[];_.each(e.streamConnections,function(e){i.push(Util._streamId(e)),n.push(Util._nodeId(e))});var r={};_.each(this.oldSources,_.bind(function(t){_.contains(i,Util._streamId(t))&&_.contains(n,Util._nodeId(t))||(console.log("[Subscriber] ",t," is no longer in the discovery results. Going to be removed."),_.each(this.nodes,_.bind(function(e){e.hasStream(t.group,t.stream)&&(console.log("[Subscriber] Remove old stream ",t," from Data Node ",e.id," dataNode counter ",e.counter),e.removeStream(t.group,t.stream,!0),void 0===r[t.group]&&(r[t.group]=[]),r[t.group].push(t.stream)),_.isEmpty(r)||(console.log("[DataNode] Emitting consumption error after removing streams"),this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR,e,r))},this)))},this))}_.each(e.streamConnections,_.bind(function(e){if(!this._exist(e)||this._error(e)||this._hasUpdatedConnection(e)){var t=Util._nodeId(e);this._hasNode(t)?(null==o[t]&&(o[t]=[]),o[t].push(e)):(null==s[t]&&(s[t]=[]),s[t].push(e))}},this)),this._connectUpdate(s),this._connectUpdate(o)},_reconnectStreams:function(e){var s={},o={};_.each(this.discoverer.getSources().streamConnections,_.bind(function(e){if(!this._exist(e)||this._error(e)&&this._hasUpdatedConnection(e)){console.log("Reconnecting to stopped stream ",e);var t=Util._nodeId(e);this._hasNode(t)?(null==o[t]&&(o[t]=[]),o[t].push(e)):(null==s[t]&&(s[t]=[]),s[t].push(e))}},this)),this._connectUpdate(s),this._connectUpdate(o)},onNodeReconnectionError:function(e,t){t==this.config().exclusionThreshold&&(this.discoverer.excluding(e.id),this._updateSourcesOnNodeReconnectionError(),console.log("Excluding node ",e.id," from discovery"))},_updateSourcesOnNodeReconnectionError:function(){0==this.hasUsedCurrentSource&&0==_.isEmpty(this.currentAvailableSources)&&(this._updateSources(this.currentAvailableSources),this.hasUsedCurrentSources=!0)},onNodeStateChanged:function(e,t){if(e.state==DataNode.State.DESTROYED||e.state==DataNode.State.DISCONNECTED){for(var s in this.consuming)this.consuming.hasOwnProperty(s)&&this.consuming[s].node.id==e.id&&(this.consuming[s].state=this.consumptionState.ERROR);var o={};for(var s in e.streams)if(e.streams.hasOwnProperty(s))for(streamInfo in o[s]=[],e.streams[s])o[s].push(streamInfo.stream);_.isEmpty(o)||(console.log("[Subscriber] emitting consumption error due to data node failed.",e.state),this.dispatch(new Subscriber.Event.ConsumptionError(e.client,o)))}t===DataNode.State.DISCONNECTED&&e.state===DataNode.State.CONNECTED?(this.discoverer.removeExclusion(e.id),console.log("Node ",e.id," resume successfully. Remove it from exclusion.")):e.state===DataNode.State.DESTROYED&&(this.discoverer.removeExclusion(e.id),delete this.nodes[e.id],console.log("Node ",e.id," is destroyed and removed from subscriber!"))},onStreamConsumption:function(e,t){this.dispatch(new Subscriber.Event.ConsumptionStart(e.client,t))},onStreamError:function(e,t){this.dispatch(new Subscriber.Event.ConsumptionError(e.client,t))},onStreamStop:function(e,t){this.dispatch(new Subscriber.Event.ConsumptionStop(e.client,t))},_discovererError:function(e){},_connectUpdate:function(e){_.each(e,_.bind(function(e,i){_.each(e,_.bind(function(t){var e=Util._streamId(t);this.consuming[e]||(this.consuming[e]=new StreamConsumptionWrapper);var s=this.consuming[e],o=Promise.resolve(0);void 0!==s&&null!=s.node&&s.state==this.consumptionState.ERROR&&(o=s.node.removeStream(t.group,t.stream,!1)),o.done=function(e,t){this.then(e,t).catch(function(e){console.log("Promise Rejected: "+e+"\n"+e.stack)})},s.state=this.consumptionState.PENDING,this.nodes[i]?o.done(_.bind(function(e){this.nodes[i].consumeStream(t),s.node=this.nodes[i]},this)):(this.nodes[i]=new DataNode(i,this.clientId,this.ticket,t,this.eventBus,this.config()),o.done(_.bind(function(e){s.node=this.nodes[i],this.nodes[i].connect()},this)))},this))},this))},_exist:function(e){var t=Util._streamId(e);return!!this.consuming[t]},_error:function(e){var t=Util._streamId(e);return!(!this.consuming[t]||this.consuming[t].state!=this.consumptionState.ERROR)},_hasUpdatedConnection:function(e){var t=Util._streamId(e);return this.consuming[t]&&this.consuming[t].node&&this.consuming[t].node.state!=DataNode.State.DESTROYED?Util._nodeId(e)!=this.consuming[t].node.id&&(console.log("Has updated source because the new node id is different from previous!"),!0):(console.log("Has updated source"),this.consuming[t]?this.consuming[t].node?this.consuming[t].node==DataNode.State.DESTROYED&&console.log("Because the source's node does not exist"):console.log("Because the source is not consumed from a node"):console.log("Because the source is not consuming"),!0)},_hasNode:function(e){return void 0!==this.nodes[e]}},IEndPointEventSource),module.exports=exports={Subscriber:Subscriber,SubscriberConfig:SubscriberConfig};
+var coreMsges = require("./CoreMessages")
+var deliveryStack = require("./DeliveryStack")
+var _ = require("underscore")
+var EventEmitter = require("eventemitter3");
+
+var streamFinder = require("@jayesol/jayeson.lib.streamfinder");
+var Discoverer = streamFinder.Discoverer;
+
+var util = require("./util")
+var _c = util._c
+
+var DefaultClient = deliveryStack.DefaultClient
+var EPEvent = deliveryStack.EPEvent
+var EPConnectedEvent = deliveryStack.EPConnectedEvent
+var EPConnectionError = deliveryStack.EPConnectionError
+var EPDisconnectedEvent = deliveryStack.EPDisconnectedEvent
+var IEndPointEventSource = deliveryStack.IEndPointEventSource
+var generateUUID = deliveryStack.generateUUID;
+var Util = deliveryStack.Util;
+var MessageWrapper = deliveryStack.MessageWrapper;
+
+var StreamRegistryMessageGroup = coreMsges.StreamRegistryMessageGroup;
+var	EmptyMessageClass = coreMsges.EmptyMessageClass;
+var	StringMessageClass = coreMsges.StringMessageClass;
+var	JSonMessageClass = coreMsges.JSonMessageClass;
+var	StreamRegistryRequest = coreMsges.StreamRegistryRequest;
+var	StreamRegistryResponse = coreMsges.StreamRegistryResponse;
+var	AuthGroup = coreMsges.AuthGroup;
+var	GenericMessageGroup = coreMsges.GenericMessageGroup;
+var	KeepAliveMessageGroup = coreMsges.KeepAliveMessageGroup;
+var TicketRenew = coreMsges.TicketRenew;
+
+/**
+ * @author Ryan
+ * 
+ * This depends on external libraries: underscorejs, eventemitter, promisejs
+ * 
+ * This depends on internal libraries: coremessage, deliverystack, jsdiscoverer
+ * 
+ * The subscriber has the following features:
+ * TODO: FILL IN THE FEATURES 
+ * 
+ */
+
+/**
+ * Represents a configuration being passed
+ * into the subscriber. In the javascript context,
+ * 
+ * The discovery pattern has the following format
+ * {
+ * 		id1 : [pattern 1, pattern 2 .... ],
+ * 		id2 : [pattern A, pattern B .... ]			
+ * }
+ * 
+ * example
+ * 
+ * {
+ * 		8 : ["*_LIVE", "SBO_*"],	// price feed
+ * 		7 : ["order_ryan"]			// order data feed for user ryan
+ * }
+ * 
+ * The processor setup is as following:
+ * {
+ * 		id1 : processor 1 of type IMessageGroupProcessor,
+ * 		id2 : processor 2 of type IMessageGroupProcessor
+ * 
+ * }
+ * 
+ */
+function SubscriberConfig() {
+	this.autoDiscover = true;	// by default, enable auto discovery
+	this.streamFinder = "";		// the URL to connect to the stream finder
+	this.username = "";			// if we want to login directly to the stream finder
+	this.password = "";
+	this.discoveryPattern = {};
+	this.processors = {};
+	this.schemes = ["ws"];		// by default we consume from ws
+	this.ticketRenewalInterval = 14*60*1000;	// default to 14 minutes
+	this.reconnectionInterval = 3000;			// reconnection to a relay node
+	this.exclusionThreshold = 3;				// the number of reconnection failed before the relay node is added to the excluded list
+	this.feedScope = "";
+	this.discoverByScope = true; //new version will discover by scope
+}
+
+/**
+ * Represents a data node. This encapsulate the following information
+ * <li>The client used to connect to the node</li>
+ * <li>The stream being consumed from the node</li>
+ * 
+ * It also provides functionalities such as reconnection. It listen to ticket being renewed
+ * by the subscriber and send the ticket to the server accordingly.
+ * 
+ * @params
+ * 	<li>id - id of the data node</li>
+ * 	<li>streamInfo - the first stream information used for connection<br/>
+ *	<pre> 
+ * 	{
+ *		"group": "<groupId>",
+ *		"stream": "<streamName>",
+ *		"level": <level>,
+ *		"serviceId" : <serviceId>,
+ *		"connection": {
+ *   		"protocol": "<protocol>",
+ *   		"hostname": "<hostname>",
+ *   		"port": <port>
+ * 		}
+ *	}
+ *	</pre>
+ *	</li>
+ *	<li>clientId - id of the client</li>
+ *	<li>ticket - ticket of the subscriber</li>
+ *	<li>eventBus - an Event Emitter which is responsible to propagate events from DataNode</li>
+ *	<li>subscriberConfig - configuration of the subscriber that created this data node</li>
+ */
+var DATA_NODE_COUNTER = 0;
+
+function DataNode(id, clientId, ticket, streamInfo, eventBus, subscriberConfig) {
+	this.client = null;
+	/**
+	 * This data structure has the following form:
+	 * { group -> [stream information] }
+	 */
+	this.streams = {};
+	this.id = id;		// id of the data node
+	
+	this.counter = DATA_NODE_COUNTER++;
+	console.log("[DataNode] Node ", id, " counter ", this.counter, " is created!");
+	
+	/**
+	 * Current state of the data node
+	 */
+	this.state = DataNode.State.CREATED;
+	this.connection = streamInfo.connection;
+	
+	this.toBeConsumed = [];
+	
+	this.consumeStream(streamInfo);
+	
+	this.clientId = clientId;
+	this.ticket = ticket;
+	
+	this.consumeTaskTimeoutID = 0;
+	
+	this.eventBus = eventBus;
+	
+	this.sConf = subscriberConfig;
+	
+	// the list of promises which will be resolved when stop consumption messages
+	// for a stream is handled and replied by the server. The promise
+	// shall be successful if there is disconnection while waiting for the reply
+	// Format: streamId -> promiseObject e.g. "8_CROWN"
+	// the promise value is true / false
+	this.removalPromises = {};
+	// the list of promises which will be resolved when consumption messages
+	// for a stream is handled and replied by the server. The promise shall
+	// resolve to failed if there is disconnection waiting for the reply. The promise
+	// is created when the registration (not consumption) for the stream starts.
+	// The promise value is true / false.
+	this.consumptionPromises = {};
+	
+};
+DataNode.State = {
+	CREATED : 1,
+	CONNECTING : 2,
+	CONNECTED : 3,
+	RECONNECTING : 4,
+	DISCONNECTED : 5,
+	DESTROYED : 6
+};
+DataNode.Event = {
+	STATE : "DN_STATE",
+	RECONNECTION_ERROR : "DN_RE_ERROR",
+	CONSUMPTION_ERROR : "DN_CONSUMPTION_ERROR",
+	CONSUMPTION_START : "DN_CONSUMPTION_START",
+	CONSUMPTION_STOP : "DN_CONSUMPTION_STOP",
+};
+_c(DataNode, {
+	
+	/**
+	 * This method return a PROMISE of the removal of the stream from the 
+	 * from consumption. This has to be asynchronous because involving sending
+	 * deregistration message to the stream registry of the consuming node. We however, only
+	 * send unregister when the node is in CONNECTED STATE and the person invoke this method
+	 * specify he want to do so (sendDeregistration = true)
+	 * 
+	 * There are two cases that a stream can be removed from a data node:
+	 * <li>The node that consuming the stream has an error and needs reconnection</li>
+	 * <li>A better node can be found for the stream according to the stream selection strategy</li>
+	 * 
+	 * @return a promise which is resolved when the stream is completely removed from consumption.
+	 */
+	removeStream : function(group, stream, sendDeregistration) {
+		
+		var id = Util._streamIdFromNames(group, stream);
+		
+		var deregistrationAction = _.bind(function(registrationResult) {
+			// the unregistration promise. This is done when the
+			// the stream is unregister from this data node by sending message to
+			// the data source
+			var promise;
+			
+			// the action to be done, remove the stream from the array
+			var unregisterPostAction = _.bind(function(result) {
+				if (result && this.streams[group]) {
+					var index;
+					
+					for (index = 0; index < this.streams[group].length; index++)
+						if (stream == this.streams[group][index].stream) break;
+					
+					if (index < this.streams[group].length) this.streams[group].splice(index, 1);
+					
+					if (this.streams[group].length == 0) delete this.streams[group];
+					
+					// if there is no more stream this node is consuming, just destroy itself
+					if (_.isEmpty(this.streams)) this.destroy();
+				}
+				
+				return result;
+			}, this);
+			
+			if (this.state == DataNode.State.CONNECTED && sendDeregistration) {
+				
+				
+				promise = Util._newPromise();
+				
+				
+				this.removalPromises[id] = promise;
+				
+				var req = new StreamRegistryRequest(group, [stream], StreamRegistryRequest.DEREGISTRATION);
+				this.client.send(new MessageWrapper(req, StreamRegistryMessageGroup.REQUEST));
+
+				var self = this;
+				window.setTimeout(function() {
+					//proceed with deregistration if no deregistration responses are received.
+					if (self.removalPromises[id] !== undefined) {
+						console.warn("Timeout waiting for deregistration response, proceeding to treat stream " + group + ":" + stream + " as deregistered for " + self.connection.hostname + ":" + self.connection.port);
+						var streamResponse = {};
+						streamResponse[stream] = StreamRegistryResponse.DEREGISTRATION_TIMEOUT;
+						var response = new StreamRegistryResponse(group, streamResponse, StreamRegistryResponse.DEREGISTRATION);
+
+						self._processDeregistrationResult(response);
+					}
+				}, 7000);
+				
+				// chain the promise
+				return promise.then(unregisterPostAction);
+				
+			} else {
+				// if the DataNode is not in connected state, no need to send anything!
+				promise = Promise.resolve(unregisterPostAction(true));
+				return promise;
+			}
+		}, this);
+		
+		// if there is a consumption in progress, wait for it! We will de-register the stream after the it finish consumption
+		// de-registration only happens in failing over or switching to better stream, so it's okay to de-register
+		// even when the pending consumption is successful. We switch anyway.
+		if (this.consumptionPromises && this.consumptionPromises[id] !== undefined) {	
+			return this.consumptionPromises[id].then(deregistrationAction);
+		} else {
+			return deregistrationAction(true);
+		}
+
+	},
+	
+	hasStream : function(group, stream) {
+		if (!this.streams[group]) {
+			return false;
+		}
+		var stream = _.each(this.streams[group], function(node) { return node.stream == stream; } );
+		return stream !== undefined;
+	},
+	
+	/**
+	 * Start consumption of a particular stream by putting it into tobeConsumed queue.
+	 * When a stream is consumed successfully, DataNode will emit an event
+	 * so that other parties know about it.
+	 */
+	consumeStream : function(streamInfo) {
+		
+		console.log("Node "+this.id+" is going to consume "+streamInfo.group+"_"+streamInfo.stream);
+		
+		this.toBeConsumed.push(streamInfo);
+		
+		if (!this.streams[streamInfo.group]) this.streams[streamInfo.group] = [];
+		
+		this.streams[streamInfo.group].push(streamInfo);	// put the stream into the list of streams that this DataNode handling
+		
+		if (this.consumeTaskTimeoutID == 0 && this.state == DataNode.State.CONNECTED)
+			this.consumeTaskTimeoutID = window.setTimeout(_.bind(this.consumeTask, this), 100);
+	},
+	
+	_consumeStreamsOnReconnected : function(){
+		
+		if (_.isEmpty(this.streams)) this.destroy();
+		
+		for(var group in this.streams){
+			if(this.streams.hasOwnProperty(group)){
+				for(var i=0 ; i<this.streams[group].length; i++){
+					this.toBeConsumed.push(this.streams[group][i]);
+				}
+			}
+		}
+	},
+	
+	connect : function() {
+		
+		console.log("[DataNode] Id ", this.id, " counter ", this.counter, " is going to connect. Current state ", this.state);
+		
+		//temp code
+		//this.connection.hostname = "local.vodds.jayeson.com.sg";
+		//this.connection.port = 5000;
+
+		this.client = new DefaultClient({
+			url : this._getUrl(this.connection),
+			msgClass : AuthGroup.classById(1),
+			params : {
+				clientId : this.clientId,
+				ticket : this.ticket,
+				feedScope : this.sConf.feedScope
+			}
+		}, KeepAliveMessageGroup);
+		this.client.attachListener(this);
+		this.client.enableKeepAlive();
+		this.client.registerGroup(AuthGroup, this);
+		this.client.registerGroup(StreamRegistryMessageGroup, this);
+		
+		this.client.connect();
+	},
+	
+	reconnect : function() {
+		
+		console.log("[DataNode] Id ", this.id, " counter ", this.counter, " is going to re-connect. Current state ", this.state);
+		
+		// update the parameters to be updated when reconnect
+		// the ticket might have changed!!
+		this.client.config.params = {
+			clientId : this.clientId,
+			ticket : this.ticket,
+			feedScope : this.sConf.feedScope
+		}
+		this.client.connect();
+	},
+	
+	_getUrl : function (connection) {
+		return connection.protocol+"://"+connection.hostname+((connection.port !== undefined)?(":"+connection.port):"")+((connection.path !== undefined)?connection.path:"");
+	},
+	
+	/**
+	 * check the consume queue and do registration one by one
+	 */
+	consumeTask : function () {
+		
+		var self = this;
+		
+		if (this.state == DataNode.State.CONNECTED) {
+			var streamByGroup = _.groupBy(this.toBeConsumed, function(streamInfo) { return  streamInfo.group; } );
+			
+			// javascript is single threaded so no need to worry about synchronization here!!
+			this.toBeConsumed = [];			// clear the toBeConsumed list.
+			this.consumeTaskTimeoutID = 0;	// reset the task handler
+			
+			// for each group to be consumed, create a promise, send a message
+			// the promise will be resolved by the onMessage method, or when disconnection
+			_.each(streamByGroup, function(streams, group) {
+				
+				// if no one is removing the stream, then we can consume it
+				// this is to avoid double consumption from two different node
+				// because if removal can only be the result of failing over
+				// or switching to a better source
+				// if the stream is being in the process of consuming, we don't
+				// resend registration as well
+				var canConsume = _.filter(streams, function(streamInfo) {
+					var sid = Util._streamId(streamInfo);
+					
+					return self.removalPromises[sid] === undefined && 
+							self.consumptionPromises[sid] === undefined; 
+				});
+				
+				// we have to create one promise for each stream whose registration
+				// is going to be sent, because we handle at stream level
+				// store those promises to the consuming list
+				var promises = _.reduce(canConsume, function(memo, streamInfo) {
+					
+					var sid = Util._streamId(streamInfo);
+					memo[sid] = Util._newPromise();
+					memo[sid].then(function(result) {
+						console.log("Consumption result of ", sid, " : successful = ",result);
+					});
+					return memo;
+					
+				}, self.consumptionPromises);
+				
+				// send the registration messages
+				var streamNames = _.map(canConsume, function(streamInfo) {
+					return streamInfo.stream;
+				});
+				
+				var req = new StreamRegistryRequest(group, streamNames, StreamRegistryRequest.REGISTRATION);
+				// start consumption by first registration message
+				self.client.send(new MessageWrapper(req, StreamRegistryMessageGroup.REQUEST));
+				
+			});
+		}
+		
+	}, 
+	
+	onEvent : function(event) {
+		
+		console.log("[DataNode] Event from the stack client", event);
+		
+		if (event instanceof EPConnectedEvent) {
+			if(this.state === DataNode.State.DISCONNECTED){
+				this._consumeStreamsOnReconnected();
+			}
+			this.setState(DataNode.State.CONNECTED);
+			console.log("[DataNode] Connect / Re-connect to ", this.connection.hostname,":", this.connection.port, "successfully. Consumption will be started!");
+			
+			this.consumeTaskTimeoutID = window.setTimeout(_.bind(this.consumeTask, this), 100);
+			
+		} else if (event instanceof EPDisconnectedEvent || event instanceof EPConnectionError) {
+			if (this.state == DataNode.State.DESTROYED) return;	// do nothing if the data node is destroyed.
+			
+			this.setState(DataNode.State.DISCONNECTED);
+			console.log("Connect / Re-connect to ",this.connection.hostname,":", this.connection.port, " with error. Retry again later.");
+			
+			// disconnection will result in all consumptionPromises resolve to false, and all 
+			// de-registration promises resolve to true
+			_.each(this.consumptionPromises, function(p, key) { p.resolve(false); } );
+			_.each(this.removalPromises, function(p, key) { p.resolve(true); } );
+			
+			delete this.consumptionPromises;
+			delete this.removalPromises;
+			
+			this.consumptionPromises = {};
+			this.removalPromises = {};
+			
+			if (this.state != DataNode.State.DESTROYED) window.setTimeout(_.bind(this.reconnect, this), this.sConf.reconnectionInterval);
+			else console.log("Data node ", this.id, " has been destroyed. Stop reconnecting.");
+			
+			if (this.consumeTaskTimeoutID !== 0) window.clearTimeout(this.consumeTaskTimeoutID);
+			
+			if (event instanceof EPConnectionError) this.eventBus.emit(DataNode.Event.RECONNECTION_ERROR, this, ++this.numFailedReconnection);
+			else this.numFailedReconnection = 0;
+			
+		}
+		
+	},
+	
+	process : function(messageWrapper) {
+		
+		if (messageWrapper.message instanceof StreamRegistryResponse) {
+			var rep = messageWrapper.message;
+			
+			if (rep.responseType == StreamRegistryResponse.REGISTRATION) {	// registration response
+				
+				this._processRegistrationResult(rep);
+
+			} else if (rep.responseType == StreamRegistryResponse.CONSUMPTION) {
+				
+				this._processConsumptionResult(rep);
+				
+			} else if (rep.responseType == StreamRegistryResponse.DEREGISTRATION) {
+				
+				this._processDeregistrationResult(rep);
+				
+			} else if (rep.responseType == StreamRegistryResponse.STOP_CONSUMPTION) {
+				this._processStopConsumptionResult(rep);
+			}
+		}
+		
+	},
+	
+	_resolveConsumptionPromise : function(group, stream, value) {
+		var id = Util._streamIdFromNames(group, stream);
+		if (this.consumptionPromises[id] !== undefined)
+			this.consumptionPromises[id].resolve(value);
+		
+		delete this.consumptionPromises[id];
+	},
+	
+	_resolveRemovalPromise : function(group, stream, value) {
+		var id = Util._streamIdFromNames(group, stream);
+		if (this.removalPromises[id] !== undefined) {
+			this.removalPromises[id].resolve(value);
+			delete this.removalPromises[id];
+		}
+		
+		delete this.consumptionPromises[id];
+	},
+	
+	_processDeregistrationResult : function (rep) {
+		var self = this;
+		
+		var group = rep.messageGroup;
+		var errorConsumedStreamByGroup = {}; // contain all unsuccessfully consumed streams
+		_.each(rep.streams, function(result, stream) {
+			if (result == StreamRegistryResponse.FAILED) {
+				self._resolveRemovalPromise(group, stream, false);
+			} else {
+				self._resolveRemovalPromise(group, stream, true);
+				if(errorConsumedStreamByGroup[group] === undefined){
+					errorConsumedStreamByGroup[group] = [];
+				}
+				errorConsumedStreamByGroup[group].push(stream);
+			}
+		});
+		if(_.isEmpty(errorConsumedStreamByGroup) == false){
+			console.log("[DataNode] Emitting consumption error after Deregistration")
+			this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR, this, errorConsumedStreamByGroup);
+		}
+	},
+	
+	_processStopConsumptionResult : function(rep) {
+		var self = this;
+		var group = rep.messageGroup;		
+		var stoppedStreamsByGroup = {};
+		_.each(rep.streams, function(result, stream) {
+			console.log("Stopping stream " + stream);
+			self.removeStream(group, stream, false);
+			if(stoppedStreamsByGroup[group] === undefined){
+				stoppedStreamsByGroup[group] = [];
+			}
+			stoppedStreamsByGroup[group].push(stream); 
+		});
+		if (!_.isEmpty(stoppedStreamsByGroup)) {
+			this.eventBus.emit(DataNode.Event.CONSUMPTION_STOP, this, stoppedStreamsByGroup);
+		}
+	},
+
+	_processConsumptionResult : function (rep) {
+		var self = this;
+		
+		var group = rep.messageGroup;
+		var consumedStreamByGroup = {}; // contain all successfully consumed streams
+		var errorConsumedStreamByGroup = {}; // contain all unsuccessfully consumed streams
+		
+		_.each(rep.streams, function(result, stream) {
+			if (result == StreamRegistryResponse.FAILED) {
+				self._resolveConsumptionPromise(group, stream, false);
+				self.removeStream(group, stream, true);
+				
+				if(errorConsumedStreamByGroup[group] === undefined){
+					errorConsumedStreamByGroup[group] = [];
+				}
+				
+				errorConsumedStreamByGroup[group].push(stream);
+			} else {
+				self._resolveConsumptionPromise(group, stream, true);
+				if(consumedStreamByGroup[group] === undefined){
+					consumedStreamByGroup[group] = [];
+				}
+				consumedStreamByGroup[group].push(stream); 
+			}
+		});
+		
+		if(_.isEmpty(consumedStreamByGroup) == false){
+			this.eventBus.emit(DataNode.Event.CONSUMPTION_START, this, consumedStreamByGroup);
+		}
+		if(_.isEmpty(errorConsumedStreamByGroup) == false){
+			this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR, this, errorConsumedStreamByGroup);
+		}
+
+	},
+	
+	_processRegistrationResult : function(rep) {
+		
+		var self = this;
+		var group = rep.messageGroup;
+		
+		// complete and remove any failed promise
+		var cannotConsume = _.pairs(rep.streams)
+				.filter(function(p) { return p[1] == StreamRegistryResponse.FAILED; })
+				.map(_.first);
+		
+		// remove these streams from the node
+		cannotConsume.forEach(function(streamName) {
+			self._resolveConsumptionPromise(group, streamName, false);
+			self.removeStream(group, streamName, false);	// the registration is failed, so no need to send deregistration
+		});
+				
+		var canConsume = _.pairs(rep.streams)
+				.filter(function(p) { return p[1] == StreamRegistryResponse.SUCCESS; })
+				.map(_.first)	// get the stream name
+				.filter(function(streamName) {
+					if (self.sConf.processors[group] !== undefined) {	// can consume because processor for the group is defined
+						return true;
+					} else {	// cannot consume because processor is undefined
+						self._resolveConsumptionPromise(group, streamName, false);
+						return false;
+					}
+				});
+		
+		// send consumption message, at this point, the server can be sure that the processor is ready to handle the stream
+		if (!_.isEmpty(canConsume)) {
+			
+			var processor = self.sConf.processors[group];
+			self.client.registerGroup(processor.group(), processor);	// attach processor for all the consumable group, whose processor is defined.
+			
+			var req = new StreamRegistryRequest(group, canConsume, StreamRegistryRequest.CONSUMPTION);
+			self.client.send(new MessageWrapper(req, StreamRegistryMessageGroup.REQUEST));
+		}
+		
+		if (!_.isEmpty(cannotConsume)) {
+			var eventData = {};
+			eventData[group] = cannotConsume;
+			console.log("Emitting consumption error due to registration result failed.", eventData);
+			
+			this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR, this, eventData);
+		}
+	},
+	
+	setState : function(state) {
+		
+		if (state != this.state) {
+			var oldState = this.state;
+			this.state = state;
+			this.eventBus.emit(DataNode.Event.STATE, this, oldState);
+		}
+		
+	},
+	
+	renewTicket : function(ticket) {
+		this.ticket = ticket;
+		var wrapper = new MessageWrapper(new TicketRenew(this.clientId, this.ticket), AuthGroup.TICKET_RENEW);
+		if (this.state == DataNode.State.CONNECTED) this.client.send(wrapper);
+	},
+	
+	destroy : function() {
+		this.setState(DataNode.State.DESTROYED);
+		console.log("[DataNode] Disconnect the destroyed data node.");
+		this.client.disconnect();
+	}
+	
+});
+
+function Subscriber(configuration) {
+	IEndPointEventSource.call(this);			// initial IEndPointEventSource
+	this.conf = new SubscriberConfig();
+	
+	_.extend(this.conf, configuration);		// copy the configuration over
+	
+	this.consuming = {};		// a map from group / stream to the wrapper
+	this.nodes = {};			// a map from source id to the information related to the source, e.g. state whether connecting, reconnecting, connected or the client used to connect to the node
+	
+	// store the currently available sources retrieved from the discoverer
+	// this data will be used for fast failover whenever there is some 
+	// connection error in a DataNode. 
+	this.currentAvailableSources = null;
+	// Check currentAvailableSources has used.
+	this.hasUsedCurrentSources = false;
+	
+	this.clientId = this.uuid();
+	this.ticket = null;
+	
+	this.eventBus = new EventEmitter();
+	
+	this.eventBus.on(DataNode.Event.RECONNECTION_ERROR, _.bind(this.onNodeReconnectionError, this));
+	this.eventBus.on(DataNode.Event.STATE, _.bind(this.onNodeStateChanged, this));
+	this.eventBus.on(DataNode.Event.CONSUMPTION_START, _.bind(this.onStreamConsumption,this));
+	this.eventBus.on(DataNode.Event.CONSUMPTION_ERROR, _.bind(this.onStreamError, this));
+	this.eventBus.on(DataNode.Event.CONSUMPTION_STOP, _.bind(this.onStreamStop, this));
+}
+function StreamConsumptionWrapper(state, node) {
+	this.state = state;
+	this.node = node;
+}
+
+/**
+ * Define Subscriber event and will be fired everytime a client is connected or disconnected.
+ */
+function ConsumptionStart(client, streams) {
+	EPEvent.call(this, client._ep);
+	this.client = client;
+	this.streams = streams;
+}
+function ConsumptionStop(client, streams) {
+	EPEvent.call(this, client._ep);
+	this.client = client;
+	this.streams = streams;
+}
+function ConsumptionError(client, streams) {
+	EPEvent.call(this, client._ep);
+	this.client = client;
+	this.streams = streams;
+}
+_c(ConsumptionStart, {}, EPEvent);
+_c(ConsumptionStop, {}, EPEvent);
+_c(ConsumptionError, {}, EPEvent);
+
+Subscriber.Event = {
+	ConsumptionStart : ConsumptionStart,
+	ConsumptionStop : ConsumptionStop,
+	ConsumptionError : ConsumptionError
+};
+
+_c(Subscriber, {
+	
+	consumptionState : {
+		PENDING : 1,		// a stream is in this state when it is waiting for a data node to connect and then perform registration
+		ERROR : 3			// a stream is in this state when its data node is having connection problem
+	},
+	
+	uuid : function (){
+ 	
+		return generateUUID();
+
+	},
+	
+	config : function() { return this.conf; },
+	/**
+	 * This method starts the discovery process
+	 */
+	start : function() {
+		var discoverer = new Discoverer(this.config().streamFinder);	// create a discoverer to connect to stream finder
+		if (this.conf.discoverByScope) {
+			discoverer = new Discoverer(this.config().streamFinder, undefined, this.conf.feedScope);	// discover by streamfinder scope
+		}
+		
+		// configure discovery
+		_.each(this.config().discoveryPattern, function(patternList, group) {
+			
+			_.each(patternList, function(pattern) {
+				discoverer.discover(group, pattern);
+			});
+			
+		});
+		
+		_.each(this.config().schemes, function(scheme) {
+			discoverer.using(scheme);
+		});
+		
+		discoverer.on("update", _.bind(this._updateSources, this));
+		discoverer.on("error", _.bind(this._discovererError, this));
+		setInterval(_.bind(this._reconnectStreams, this), 5000);
+		this._makeTicket(true);
+
+		this.discoverer = discoverer;
+	},
+	
+	_makeTicket : function(isStarting) {
+		SessionFactory.makeTicket(this.config().streamFinder, this.clientId, _.bind(function(ticket, error) {
+			if (ticket.length != 0) {
+				
+				this.ticket = ticket;
+				console.log("New ticket obtained "+this.ticket);
+				
+				if (isStarting) {				// if this is not renewal, we should start the discovery process
+					this.discoverer.start();
+				} else {
+					_.each(this.nodes, _.bind(function(node, id) {
+						window.setTimeout(_.bind(node.renewTicket, node, this.ticket), 0);	// asynchronously
+					}, this));
+				}
+				
+				window.setTimeout(_.bind(this._makeTicket, this), this.config().ticketRenewalInterval);
+				
+			} else if (error.length != 0) {
+				window.setTimeout(_.bind(this._makeTicket, this), 1000);
+			}
+		}, this));
+	},
+	
+	_updateSources : function(sources) {
+		
+		console.log("Updating Sources!");
+		console.log(sources);
+		
+		this.oldSources = this.currentAvailableSources;
+		this.currentAvailableSources = sources.streamConnections;
+		this.hasUsedCurrentSources = false;
+		
+		var toBeConnected = {};	// a map of new nodes to be connected
+		var toBeUpdated = {};	// a map of existing node to be updated with new group, streams
+		
+		if (this.oldSources && this.oldSources.length > 0) {
+			var streamIds = [];
+			var nodeIds = [];
+			
+			_.each(sources.streamConnections, function(source) {
+				streamIds.push(Util._streamId(source));
+				nodeIds.push(Util._nodeId(source));
+			});
+			
+			// remove from corresponding old sources not exist in new sources from data nodes
+			var stopStreamsByGroup = {};
+			_.each(this.oldSources, _.bind(function(oldSource) {
+				
+				if (!_.contains(streamIds, Util._streamId(oldSource)) || !_.contains(nodeIds, Util._nodeId(oldSource))) {
+					
+					console.log("[Subscriber] ", oldSource, " is no longer in the discovery results. Going to be removed.")
+					
+					_.each(this.nodes, _.bind(function(dataNode) {
+						
+						if (dataNode.hasStream(oldSource.group, oldSource.stream)) {
+							console.log("[Subscriber] Remove old stream ", oldSource, " from Data Node ", dataNode.id, " dataNode counter ", dataNode.counter);
+							dataNode.removeStream(oldSource.group, oldSource.stream, true);
+							if(stopStreamsByGroup[oldSource.group] === undefined){
+								stopStreamsByGroup[oldSource.group] = [];
+							}
+							stopStreamsByGroup[oldSource.group].push(oldSource.stream);
+						}
+						if(!_.isEmpty(stopStreamsByGroup)) {
+							console.log("[DataNode] Emitting consumption error after removing streams")
+							this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR, dataNode, stopStreamsByGroup);
+						}
+					}, this));
+				}
+				
+			}, this));
+			
+		}
+		
+		_.each(sources.streamConnections, _.bind(function(source) {	// for each source, we check if we are already consuming it
+			
+			/**
+			 * if we are not currently consuming a particular group & stream,
+			 * or if we are consuming it but the connection for that group & stream is having
+			 * problem, we will gather the group & stream by source id. 
+			 */
+			if (!this._exist(source) || 
+					// if there is error at the node currently consuming the stream
+					// and the connection returned by discoverer is different from the one
+					// currently use (which is causing error)
+					this._error(source) || this._hasUpdatedConnection(source)) {
+				
+				var nodeId = Util._nodeId(source);
+				
+				if (this._hasNode(nodeId)) {
+					if (toBeUpdated[nodeId] == undefined) toBeUpdated[nodeId] = [];
+					toBeUpdated[nodeId].push(source);
+				} else {
+					if (toBeConnected[nodeId] == undefined) toBeConnected[nodeId] = [];
+					toBeConnected[nodeId].push(source);
+				}
+			}
+			
+		}, this));
+		
+		// any group, and stream which are in these list should be removed from current
+		// data node first, before being added to new data node
+		this._connectUpdate(toBeConnected);
+		this._connectUpdate(toBeUpdated);
+		
+	},
+
+	_reconnectStreams : function(sources) {
+		var toBeConnected = {};	// a map of new nodes to be connected
+		var toBeUpdated = {};	// a map of existing node to be updated with new group, streams
+		_.each(this.discoverer.getSources().streamConnections, _.bind(function(source) {	// for each source, we check if we are already consuming it
+
+			/**
+			* if we are not currently consuming a particular group & stream,
+			* or if we are consuming it but the connection for that group & stream is having
+			* problem, we will gather the group & stream by source id. 
+			*/
+			if (!this._exist(source) || 
+				// if there is error at the node currently consuming the stream
+				// and the connection returned by discoverer is different from the one
+				// currently use (which is causing error)
+				(this._error(source) && this._hasUpdatedConnection(source))) {
+
+				console.log("Reconnecting to stopped stream ", source);
+				var nodeId = Util._nodeId(source);
+				if (this._hasNode(nodeId)) {
+					if (toBeUpdated[nodeId] == undefined) toBeUpdated[nodeId] = [];
+					toBeUpdated[nodeId].push(source);
+				} else {
+					if (toBeConnected[nodeId] == undefined) toBeConnected[nodeId] = [];
+					toBeConnected[nodeId].push(source);
+				}
+			}
+
+		}, this));
+		this._connectUpdate(toBeConnected);
+		this._connectUpdate(toBeUpdated);
+	},
+	
+	onNodeReconnectionError : function(dataNode, num) {
+		
+		if (num == this.config().exclusionThreshold) {
+			this.discoverer.excluding(dataNode.id);
+			this._updateSourcesOnNodeReconnectionError();
+			console.log("Excluding node ", dataNode.id, " from discovery");
+		}
+		
+	},
+	
+	_updateSourcesOnNodeReconnectionError : function(){
+		// Check if currentAvailableSources is not empty and not used.
+		if(this.hasUsedCurrentSource == false && _.isEmpty(this.currentAvailableSources) == false){
+			this._updateSources(this.currentAvailableSources);
+			this.hasUsedCurrentSources = true; // mark currentAvailableSources has used.
+		}
+	},
+	
+	onNodeStateChanged : function(dataNode, oldState) {
+		
+		// if the data node is destroyed or disconnected. Need to inform clients about the consumption error
+		// also mark those sources provided by that nodes as having error state
+		if (dataNode.state == DataNode.State.DESTROYED || dataNode.state == DataNode.State.DISCONNECTED) {
+							
+			for(var property in this.consuming){
+				if(this.consuming.hasOwnProperty(property)){
+					if(this.consuming[property].node.id == dataNode.id){
+						this.consuming[property].state = this.consumptionState.ERROR;
+					}
+				}
+			}
+			
+			var streamsByGroup = {};
+			
+			for(var property in dataNode.streams){
+				if(dataNode.streams.hasOwnProperty(property)){
+					streamsByGroup[property] = [];
+					for (streamInfo in dataNode.streams[property])
+						streamsByGroup[property].push(streamInfo.stream);
+				}
+			}
+			
+			if (!_.isEmpty(streamsByGroup)) {	// if streamsByGroup doesn't contain any thing, no need to emit event
+				console.log("[Subscriber] emitting consumption error due to data node failed.", dataNode.state);
+				this.dispatch(new Subscriber.Event.ConsumptionError(dataNode.client, streamsByGroup));
+			}
+		}
+		
+		if (oldState === DataNode.State.DISCONNECTED && dataNode.state === DataNode.State.CONNECTED) {
+			this.discoverer.removeExclusion(dataNode.id);
+			console.log("Node ", dataNode.id, " resume successfully. Remove it from exclusion.");
+		} else if (dataNode.state === DataNode.State.DESTROYED) {
+			// when it's destroyed, we are not going to reconnect to it automatically, so remove from exclusion
+			this.discoverer.removeExclusion(dataNode.id);
+			delete this.nodes[dataNode.id];
+			console.log("Node ", dataNode.id, " is destroyed and removed from subscriber!");
+			
+		}
+		
+	},
+
+	onStreamConsumption : function(dataNode, streamsByGroup){
+		this.dispatch(new Subscriber.Event.ConsumptionStart(dataNode.client, streamsByGroup));
+	},
+	
+	onStreamError : function(dataNode, streamsByGroup){
+		this.dispatch(new Subscriber.Event.ConsumptionError(dataNode.client, streamsByGroup));
+	},
+
+	onStreamStop : function(dataNode, streamsByGroup){
+		this.dispatch(new Subscriber.Event.ConsumptionStop(dataNode.client, streamsByGroup));
+	},
+	
+	_discovererError : function (err) {
+		
+	},
+	
+	/**
+	 * This method remove the group & stream which are going to be 
+	 * consumed in NEW DataNode or to be ADDED to other EXISTING
+	 * DataNodes from the current DataNodes they belong to (because current DataNodes
+	 * might be having connection error, which marks the respective group & streams
+	 * as error). Only when there is no one else consuming the stream, 
+	 * the stream can be registered into a new node. 
+	 * 
+	 * @param sources : a map from the nodeId of nodes to be added / updated to the group and stream. 
+	 */
+	_connectUpdate : function(sources) {
+		
+		_.each(sources, _.bind(function(connections, nodeId) {
+						
+			_.each(connections, _.bind(function(streamInfo) {
+				var streamId = Util._streamId(streamInfo);
+				
+				if (!this.consuming[streamId]) this.consuming[streamId] = new StreamConsumptionWrapper();
+				
+				var wrapper = this.consuming[streamId];
+				
+				var promise = Promise.resolve(0);
+				
+				// this will automatically destroy the node if there is no stream for it to consume
+				if (wrapper !== undefined && wrapper.node != undefined && wrapper.state == this.consumptionState.ERROR)
+					promise = wrapper.node.removeStream(streamInfo.group, streamInfo.stream, false);	// if consumption state is already error, no need to send deregistration
+				
+				promise.done = function(onFulfilled, onRejected) {
+					this.then(onFulfilled, onRejected)
+						.catch(function (reason) {
+							console.log("Promise Rejected: " + reason + "\n" + reason.stack);
+						});
+				};
+				
+				// this stream is going to be consume soon
+				wrapper.state = this.consumptionState.PENDING;
+				/**
+				 * when the promise is done
+				 * it means that there is no one consuming the group & stream
+				 * it's okay now that I'll start consuming it
+				 */
+				if (this.nodes[nodeId]) {	// if the node already existed
+					promise.done(_.bind(function(result) { 
+						
+						this.nodes[nodeId].consumeStream(streamInfo);
+						wrapper.node = this.nodes[nodeId];	// mark this so that no one else can consume this stream
+						
+					}, this));
+				} else {	// a new data node has to be created
+					
+					this.nodes[nodeId] = new DataNode(nodeId, this.clientId, this.ticket, streamInfo, this.eventBus, this.config());
+					promise.done(_.bind(function(result) {
+						wrapper.node = this.nodes[nodeId];	// mark this so that no one else can consume this stream
+						this.nodes[nodeId].connect();
+						
+					}, this));
+					
+				}
+				
+			}, this));
+			
+		}, this));
+		
+	},
+	
+	/**
+	 * Check if a group and stream existed
+	 * This means, the subscriber is consuming data for this group and stream from
+	 * a source.
+	 */
+	_exist : function(source) {
+		var streamId = Util._streamId(source);
+		
+		if (this.consuming[streamId]) return true;
+		
+		return false;
+	},
+	
+	/**
+	 * Check if a consuming group and stream is having any problem.
+	 * A consuming group and stream has problem if the DataNode for it is having 
+	 * connection problem.
+	 */
+	_error : function(source) {
+		var streamId = Util._streamId(source);
+		
+		if (this.consuming[streamId] && this.consuming[streamId].state == this.consumptionState.ERROR) return true;
+		
+		return false;
+	},
+	
+	/**
+	 * Check if a source returned by discoverer has newer connection
+	 * than the one being used
+	 */
+	_hasUpdatedConnection : function (source) {
+		var streamId = Util._streamId(source);
+		
+		if (!this.consuming[streamId] || !this.consuming[streamId].node || 
+				this.consuming[streamId].node.state == DataNode.State.DESTROYED) {
+			console.log("Has updated source");
+			
+			if (!this.consuming[streamId]) 
+				console.log("Because the source is not consuming");
+			else if (!this.consuming[streamId].node) 
+				console.log("Because the source is not consumed from a node");
+			else if (this.consuming[streamId].node == DataNode.State.DESTROYED) 
+				console.log("Because the source's node does not exist");
+			
+			return true;	// if this source is not in the consuming list or if it is, but there is no node for it
+		} else if (Util._nodeId(source) != this.consuming[streamId].node.id) {
+			console.log("Has updated source because the new node id is different from previous!");
+			return true;	// if the service id of the source is different from service id of the node
+		}
+		
+		return false;
+	},
+	
+	_hasNode : function(nodeId) {
+		return this.nodes[nodeId] !== undefined;
+	},
+	
+}, IEndPointEventSource);
+
+module.exports = exports = {
+	Subscriber : Subscriber,
+	SubscriberConfig : SubscriberConfig
+}